diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..753cb2195476e134b29df3b06b80f7be583a4001 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20230711_182238-k29fqnbt/run-k29fqnbt.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20230711_182238-wt8le4qg/run-wt8le4qg.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20230711_182238-zq31sjjd/run-zq31sjjd.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20230712_000308-hbsh7yzb/run-hbsh7yzb.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20230712_000308-lh5nw6i8/run-lh5nw6i8.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0415920a106502b92b8105647c1e3bf7258bc005 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ + +--- +license: creativeml-openrail-m +base_model: runwayml/stable-diffusion-v1-5 +tags: +- stable-diffusion +- stable-diffusion-diffusers +- text-to-image +- diffusers +- lora +inference: true +--- + +# LoRA text2image fine-tuning - asrimanth/person-thumbs-up-plain-lora +These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the Custom dataset dataset. You can find some example images in the following. + +![img_0](./image_0.png) +![img_1](./image_1.png) +![img_2](./image_2.png) +![img_3](./image_3.png) + diff --git a/checkpoint-1000/optimizer.bin b/checkpoint-1000/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..5abdc2d1e38d5670ae41f9ba09c3330fad39b1cf --- /dev/null +++ b/checkpoint-1000/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d6cc84384435472775b08eb1b36db0e391123d01838f0bf629e2a6903872d7 +size 6591685 diff --git a/checkpoint-1000/pytorch_model.bin b/checkpoint-1000/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..232407c218019f61b84cd50a2660fb77ea791060 --- /dev/null +++ b/checkpoint-1000/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42e0eb5b8867df808ad9c97d3e100db59d7a3c5dd829b4390ef4e78ea0eb56cb +size 3285965 diff --git a/checkpoint-1000/random_states_0.pkl b/checkpoint-1000/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..97d65b98e5983a526d9cbe21e43bd4708c33a825 --- /dev/null +++ b/checkpoint-1000/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70f46034afce00e501c95a101d672135d81d8b659749abecb24ea30660bb64f2 +size 16719 diff --git a/checkpoint-1000/scaler.pt b/checkpoint-1000/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..efdbd3c795f6b0d4144e68355e99c220ccdedd09 --- /dev/null +++ b/checkpoint-1000/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68cff80b680ddf6e7abbef98b5f336b97f9b5963e2209307f639383870e8cc71 +size 557 diff --git a/checkpoint-1000/scheduler.bin b/checkpoint-1000/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..a8f0ff3ec3672f2517ee57c87da21e722693d390 --- /dev/null +++ b/checkpoint-1000/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4b7a8c9615fae491ac411adbefa23ee4ae0a27ebb4cf4f6f55a147af73f533c +size 563 diff --git a/checkpoint-1500/optimizer.bin b/checkpoint-1500/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..43740526eb245c6380e3c2f02c51764bee564453 --- /dev/null +++ b/checkpoint-1500/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0c4c30117df639f85bc2f99d27765eff32414af8f41a52e6ef9c63e3429216 +size 6591685 diff --git a/checkpoint-1500/pytorch_model.bin b/checkpoint-1500/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..71283f490281d532a0c38030dd1540c8d0212d44 --- /dev/null +++ b/checkpoint-1500/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:536da8c93d88a2239aff4391d558d214de69e19b44032d69c8ee4c9b74af5c61 +size 3285965 diff --git a/checkpoint-1500/random_states_0.pkl b/checkpoint-1500/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..0d4001915dbb0a72f5ec2ca8c4a967d7bec04003 --- /dev/null +++ b/checkpoint-1500/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9310e30cf8e27e0d708811d592605f0ccdf6429e6bb14aa308d1444c87992b06 +size 16719 diff --git a/checkpoint-1500/scaler.pt b/checkpoint-1500/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..69c76b8f27f096d1d1a3d9d0e387af1a9cf5028d --- /dev/null +++ b/checkpoint-1500/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203a72d6c29f42a0e2964fdddc8d7a98df1eccee78fea9de0fa416613390f5c6 +size 557 diff --git a/checkpoint-1500/scheduler.bin b/checkpoint-1500/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..a0e46f83613cb2b860d197097fce940398735be3 --- /dev/null +++ b/checkpoint-1500/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:618f752fb0c91576dd5a9070cf872ce0234171aa96bd3ff7a24b782d2bc652d4 +size 563 diff --git a/checkpoint-2000/optimizer.bin b/checkpoint-2000/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..5593fc91c18575c19e48428b4bfd7874f7fb24e9 --- /dev/null +++ b/checkpoint-2000/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8aa73d11f31e98f851dc5ac6195bcdbab6a439a8245c0435cc4d2344c480b6 +size 6591685 diff --git a/checkpoint-2000/pytorch_model.bin b/checkpoint-2000/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..d9ea6ec0f5d46a663e41a40735c370befc7205f2 --- /dev/null +++ b/checkpoint-2000/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52e8292631286ee159204a56501523d48fde5bcd6904a0c88eabee6c59ad6072 +size 3285965 diff --git a/checkpoint-2000/random_states_0.pkl b/checkpoint-2000/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..bd080ab31e91e0027963739c351c5f7d77075cbb --- /dev/null +++ b/checkpoint-2000/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ed377c06c2b574bdf491162327cf3b3094dfc0ba0fcff6006b15ce6bd4ab280 +size 16719 diff --git a/checkpoint-2000/scaler.pt b/checkpoint-2000/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..21ece35416ce79724d347155f11cfa297b97cabc --- /dev/null +++ b/checkpoint-2000/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2de9749828adacdf103bf6e9592702bb7067a2c1df27dd62ab38c1eb8c070f +size 557 diff --git a/checkpoint-2000/scheduler.bin b/checkpoint-2000/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..db0aaf96a9fb942f787cad981a83d768287c5877 --- /dev/null +++ b/checkpoint-2000/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92ee0753920a1f5ae4ebe7eef9144f5df7cc16277682716fb4b4b52854d5d98c +size 563 diff --git a/checkpoint-2500/optimizer.bin b/checkpoint-2500/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..2755d4e0de583a4056ed5a34ffe64c3ced56bd99 --- /dev/null +++ b/checkpoint-2500/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aace201ed4c097aee4ba690254c0ee33a1818d820fb3f789297373c1af06090e +size 6591685 diff --git a/checkpoint-2500/pytorch_model.bin b/checkpoint-2500/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..c46e36f0e5b2932facb20a300749f5c3ca77b540 --- /dev/null +++ b/checkpoint-2500/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99be5af0c0bb2681917de4e5ac8161e7f0b94f73e40986c2ddf320f522873a3 +size 3285965 diff --git a/checkpoint-2500/random_states_0.pkl b/checkpoint-2500/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8d2f335dcb2179bffd80e7f79e5ba4b1c3301c21 --- /dev/null +++ b/checkpoint-2500/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff321b0b6ea3c26b8f06e18a53f4eb0ea8eb6b2ff40b46a5189347385c98b8e1 +size 16719 diff --git a/checkpoint-2500/scaler.pt b/checkpoint-2500/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a3e6e139e26b18feddbe2f66ba98b73890dacfa --- /dev/null +++ b/checkpoint-2500/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fbcebc8f5487b0c117b5dd47f2ea304af3eebf408d297118d9307e1223927e1 +size 557 diff --git a/checkpoint-2500/scheduler.bin b/checkpoint-2500/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..d411cfded9c14b2267300ecb09348269016f888e --- /dev/null +++ b/checkpoint-2500/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb85608cdf9aeee6120a6bf917659b4cb4b80d07eaa66820bdf03b4aff3e77fd +size 563 diff --git a/checkpoint-3000/optimizer.bin b/checkpoint-3000/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..5b6abbf488d53fb55ddbd97b10013bd0782c1723 --- /dev/null +++ b/checkpoint-3000/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a053098abecdf72c7a212c14613d470fa8c567a6767218f7351747f8a04114 +size 6591685 diff --git a/checkpoint-3000/pytorch_model.bin b/checkpoint-3000/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..a3106ab5e7d17753386ef824b3b0233d14580084 --- /dev/null +++ b/checkpoint-3000/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ab9b129ce2eef2ebf283a95fa50f0d33a2de9b70085615b6df7703da78f6ad +size 3285965 diff --git a/checkpoint-3000/random_states_0.pkl b/checkpoint-3000/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5ac4dadc45d5d54491d939fe3fba8f2115d4ef22 --- /dev/null +++ b/checkpoint-3000/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07d2ac0cf64a1c5a61faa6e5407f98acb5da02355609ddc4c73f8e60b7acc6a4 +size 16719 diff --git a/checkpoint-3000/scaler.pt b/checkpoint-3000/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2888c9ddc2c30e6b47ce0e6abcc8e2d1303aab00 --- /dev/null +++ b/checkpoint-3000/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb1f9398b77268202e8e1465734a63d123b1ef11c27f20f2473677e9883a6869 +size 557 diff --git a/checkpoint-3000/scheduler.bin b/checkpoint-3000/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..0a380ad22372f1b83ae2f3fa5116b136b2623369 --- /dev/null +++ b/checkpoint-3000/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9405eb5fb0dbe2ed341f260e11bc4bfc31a90415d8dc221362ffa787977ae3e +size 563 diff --git a/checkpoint-500/optimizer.bin b/checkpoint-500/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..d3cb3310727e9ed3928ddbaef7386ee38060547d --- /dev/null +++ b/checkpoint-500/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97de3df572cb68542d73c9d0cd45952f59168570f52da0f1b970b84fb9f46a0e +size 6591685 diff --git a/checkpoint-500/pytorch_model.bin b/checkpoint-500/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..a78488d44196d0a5d7d0603aa98cbc867f4bd77d --- /dev/null +++ b/checkpoint-500/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f19b17fb31608556888dbf87e3fb7166a6c9d040762a209c9b650b62c411d866 +size 3285965 diff --git a/checkpoint-500/random_states_0.pkl b/checkpoint-500/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3e7136063416afd693e37a40b0eb5f424fdbf395 --- /dev/null +++ b/checkpoint-500/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8892bf79d8077ede2fc4c5c130f413885e91a60cdfb8b857835e8053e0b4ff6c +size 16719 diff --git a/checkpoint-500/scaler.pt b/checkpoint-500/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5f2e5ee77c2cef5eff816170558ce9209855685 --- /dev/null +++ b/checkpoint-500/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f196a54202bb4ba1220e8c59f42f9cda0702d68ea83147d814c2fb2f36b8f2 +size 557 diff --git a/checkpoint-500/scheduler.bin b/checkpoint-500/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..6ec9d0ace2aba80b55dddc5b30534f5e339e6d49 --- /dev/null +++ b/checkpoint-500/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd3bf9a7a78805702b59d14af28a11ce03c798911f2afca318eff01a88547f4d +size 563 diff --git a/image_0.png b/image_0.png new file mode 100644 index 0000000000000000000000000000000000000000..89cfdaf8479299106959d5bc272c646d924b9d05 Binary files /dev/null and b/image_0.png differ diff --git a/image_1.png b/image_1.png new file mode 100644 index 0000000000000000000000000000000000000000..b0a0f0163817a201a0a4087136adca469edcb2f1 Binary files /dev/null and b/image_1.png differ diff --git a/image_2.png b/image_2.png new file mode 100644 index 0000000000000000000000000000000000000000..b1c159063d2559d9d4a17673042c51e7aee94c1e Binary files /dev/null and b/image_2.png differ diff --git a/image_3.png b/image_3.png new file mode 100644 index 0000000000000000000000000000000000000000..d96ca1e7534d6615a2d5393ba709e1fd5f23af44 Binary files /dev/null and b/image_3.png differ diff --git a/pytorch_lora_weights.bin b/pytorch_lora_weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..7af18212a95d5d3d0859e5b1baa6803a945b057a --- /dev/null +++ b/pytorch_lora_weights.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20fe96ddaded71bc7f7465dfa40a7dedf7977d68870b5f2cd15d159d2ddcaa3e +size 3287771 diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6464d9356fb67e4efb88916b04f4d38594c5db02 --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,46257 @@ +2023-07-12 00:03:08,088 INFO StreamThr :3148485 [internal.py:wandb_internal():86] W&B internal server running at pid: 3148485, started at: 2023-07-12 00:03:08.087197 +2023-07-12 00:03:08,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status +2023-07-12 00:03:08,092 INFO WriterThread:3148485 [datastore.py:open_for_write():85] open: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb +2023-07-12 00:03:08,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: header +2023-07-12 00:03:08,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: run +2023-07-12 00:03:08,316 INFO SenderThread:3148485 [dir_watcher.py:__init__():211] watching files in: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files +2023-07-12 00:03:08,317 INFO SenderThread:3148485 [sender.py:_start_run_threads():1100] run started: yuype6tc with start time 1689134588.087926 +2023-07-12 00:03:08,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:03:08,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: check_version +2023-07-12 00:03:08,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:03:08,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: check_version +2023-07-12 00:03:08,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: run_start +2023-07-12 00:03:08,457 DEBUG HandlerThread:3148485 [system_info.py:__init__():31] System info init +2023-07-12 00:03:08,457 DEBUG HandlerThread:3148485 [system_info.py:__init__():46] System info init done +2023-07-12 00:03:08,457 INFO HandlerThread:3148485 [system_monitor.py:start():181] Starting system monitor +2023-07-12 00:03:08,457 INFO SystemMonitor:3148485 [system_monitor.py:_start():145] Starting system asset monitoring threads +2023-07-12 00:03:08,459 INFO HandlerThread:3148485 [system_monitor.py:probe():201] Collecting system info +2023-07-12 00:03:08,459 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started cpu monitoring +2023-07-12 00:03:08,460 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started disk monitoring +2023-07-12 00:03:08,462 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started gpu monitoring +2023-07-12 00:03:08,462 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started memory monitoring +2023-07-12 00:03:08,463 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started network monitoring +2023-07-12 00:03:08,519 DEBUG HandlerThread:3148485 [system_info.py:probe():195] Probing system +2023-07-12 00:03:08,528 DEBUG HandlerThread:3148485 [gitlib.py:_init_repo():53] git repository is invalid +2023-07-12 00:03:08,528 DEBUG HandlerThread:3148485 [system_info.py:probe():240] Probing system done +2023-07-12 00:03:08,529 DEBUG HandlerThread:3148485 [system_monitor.py:probe():210] {'os': 'Linux-5.15.0-60-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2023-07-12T04:03:08.519507', 'startedAt': '2023-07-12T04:03:08.061688', 'docker': None, 'cuda': None, 'args': ('--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5', '--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/', '--resolution=512', '--center_crop', '--random_flip', '--train_batch_size=2', '--gradient_accumulation_steps=4', '--num_train_epochs=300', '--learning_rate=1e-5', '--max_grad_norm=1', '--lr_scheduler=cosine', '--lr_warmup_steps=500', '--output_dir=/l/vision/v5/sragas/easel_ai/models_plain/', '--checkpointing_steps=500', '--report_to=wandb', '--validation_prompt=tom cruise thumbs up', '--seed=51', '--push_to_hub', '--hub_model_id=person-thumbs-up-plain-lora'), 'state': 'running', 'program': 'train_text_to_image_lora.py', 'codePath': 'train_text_to_image_lora.py', 'host': 'magneton', 'username': 'sragas', 'executable': '/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3', 'cpu_count': 6, 'cpu_count_logical': 12, 'cpu_freq': {'current': 1779.2625, 'min': 1200.0, 'max': 3700.0}, 'cpu_freq_per_core': [{'current': 3100.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1500.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}, {'current': 2100.0, 'min': 1200.0, 'max': 3700.0}, {'current': 3100.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1742.515, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1300.0, 'min': 1200.0, 'max': 3700.0}, {'current': 2400.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}, {'current': 2500.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}], 'disk': {'total': 467.8895797729492, 'used': 42.768035888671875}, 'gpu': 'NVIDIA GeForce GTX TITAN X', 'gpu_count': 3, 'gpu_devices': [{'name': 'NVIDIA GeForce GTX TITAN X', 'memory_total': 12884901888}, {'name': 'NVIDIA TITAN X (Pascal)', 'memory_total': 12884901888}, {'name': 'NVIDIA GeForce GTX TITAN X', 'memory_total': 12884901888}], 'memory': {'total': 31.240768432617188}} +2023-07-12 00:03:08,529 INFO HandlerThread:3148485 [system_monitor.py:probe():211] Finished collecting system info +2023-07-12 00:03:08,529 INFO HandlerThread:3148485 [system_monitor.py:probe():214] Publishing system info +2023-07-12 00:03:08,529 DEBUG HandlerThread:3148485 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2023-07-12 00:03:08,540 DEBUG HandlerThread:3148485 [system_info.py:_save_pip():67] Saving pip packages done +2023-07-12 00:03:08,542 INFO HandlerThread:3148485 [system_monitor.py:probe():216] Finished publishing system info +2023-07-12 00:03:08,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:03:08,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now +2023-07-12 00:03:08,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:08,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:08,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: telemetry +2023-07-12 00:03:09,042 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s7kvtbzx-wandb-metadata.json +2023-07-12 00:03:09,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/requirements.txt +2023-07-12 00:03:09,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:09,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:03:09,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json +2023-07-12 00:03:11,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:13,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:13,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:17,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:18,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: telemetry +2023-07-12 00:03:18,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:18,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: config +2023-07-12 00:03:21,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:23,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:23,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:24,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:27,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:29,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:30,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:31,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:32,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:03:33,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:35,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:35,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:37,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:38,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:38,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:41,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:41,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:03:41,229 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:03:41,229 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:03:41,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:03:41,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:03:41,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/config.yaml +2023-07-12 00:03:41,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:43,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:45,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:46,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:47,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:49,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:50,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:03:50,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:03:50,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:03:50,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:03:51,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:03:51,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:51,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:53,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:53,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:55,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:57,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:57,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:59,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:00,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:00,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:00,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:00,190 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:00,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:01,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:03,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:03,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:05,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:08,464 DEBUG SystemMonitor:3148485 [system_monitor.py:_start():159] Starting system metrics aggregation loop +2023-07-12 00:04:08,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:04:08,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:08,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:08,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:09,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:09,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:09,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:09,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:09,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:10,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:11,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:13,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:13,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:17,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:19,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:19,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:20,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:20,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:20,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:20,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:21,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:21,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:23,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:23,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:24,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:25,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:27,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:29,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:29,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:29,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:29,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:29,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:30,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:30,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:31,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:33,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:36,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:37,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:38,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:04:38,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:38,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:39,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:40,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:40,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:40,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:40,804 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:41,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:41,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:41,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:43,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:45,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:47,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:47,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:49,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:50,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:50,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:50,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:50,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:51,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:51,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:52,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:53,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:54,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:54,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:54,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:54,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:54,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:54,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:55,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:55,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:57,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:58,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:59,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:05:03,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:08,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:08,468 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:05:09,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:09,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:13,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:18,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:23,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:24,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:24,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:29,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:34,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:38,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:05:39,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:39,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:39,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:44,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:49,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:54,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:54,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:55,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:00,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:03,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:03,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:06:03,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media +2023-07-12 00:06:03,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_0838dad447e339fd0fea.png with policy now +2023-07-12 00:06:03,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_a8dfbe0ff96ea5050767.png with policy now +2023-07-12 00:06:03,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,687 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_56c2e575b2b9d92db6f0.png with policy now +2023-07-12 00:06:03,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_3f8ba95016c3d0286ac3.png with policy now +2023-07-12 00:06:03,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:03,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:03,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:03,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:03,949 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zmy0twhm-media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:04,033 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3qduvr0-media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 00:06:04,048 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1gdybl44-media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 00:06:04,093 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yfkhyw3b-media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 00:06:04,503 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s4k0p976-media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:04,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:04,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:04,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 00:06:04,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 00:06:04,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 00:06:04,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:06:05,002 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lft3007p-media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 00:06:05,004 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/db4zocz3-media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 00:06:05,036 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fs39n25q-media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 00:06:05,242 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xmns3160-media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:05,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:07,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:08,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:06:09,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:10,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:10,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:10,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:11,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:12,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:15,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:15,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:17,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:19,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:21,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:21,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:23,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:23,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:23,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:23,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:23,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:25,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:25,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:25,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:27,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:27,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:29,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:31,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:32,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:32,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:32,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:32,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:32,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:32,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:33,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:35,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:37,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:38,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:06:39,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:40,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:40,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:41,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:41,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:41,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:41,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:41,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:41,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:43,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:43,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:45,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:47,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:48,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:49,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:51,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:51,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:51,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:51,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:51,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:53,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:53,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:55,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:55,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:55,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:57,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:59,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:01,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:01,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:01,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:01,471 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:01,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:01,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:03,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:04,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:05,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:07,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:08,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:07:09,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:10,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:10,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:10,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:07:11,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:11,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:11,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:11,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:11,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:13,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:15,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:16,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:17,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:21,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:21,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:21,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:21,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:21,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:21,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:22,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:23,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:25,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:25,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:25,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:07:27,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:28,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:29,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:30,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:30,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:30,983 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:30,985 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:31,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:31,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:33,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:35,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:35,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:35,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:35,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:35,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:35,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:37,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:38,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:07:39,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:40,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:40,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:07:45,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:50,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:55,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:55,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:55,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:00,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:05,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:08,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:08:10,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:10,978 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:11,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:16,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:21,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:25,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:25,978 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:26,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:29,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_b1104143852d0cdcfdbc.png with policy now +2023-07-12 00:08:29,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_a492b392b73e859b8a77.png with policy now +2023-07-12 00:08:29,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_3ac29936b57bab3b60df.png with policy now +2023-07-12 00:08:29,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_0394766c00f144b38aaf.png with policy now +2023-07-12 00:08:30,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:30,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:08:30,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:08:30,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:08:30,257 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4tjmg87-media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 00:08:30,314 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j0fggkff-media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 00:08:30,316 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sm8pmsmk-media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 00:08:30,322 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o51jb73i-media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 00:08:30,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:08:30,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 00:08:30,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 00:08:30,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 00:08:30,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 00:08:30,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:08:31,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:31,262 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qq0fimgp-media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 00:08:31,263 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q2cpsx7h-media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 00:08:31,285 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9qsl438y-media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 00:08:31,292 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6263jl6v-media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 00:08:33,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:36,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:37,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:38,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:08:39,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:39,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:41,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:41,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:41,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:43,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:45,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:46,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:47,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:49,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:49,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:08:49,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:08:49,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:08:49,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:08:51,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:51,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:53,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:55,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:56,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:56,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:59,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:59,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:08:59,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:08:59,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:08:59,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:08:59,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:01,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:01,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:03,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:05,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:07,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:07,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:08,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:09:09,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:09,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:09,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:09,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:09,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:11,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:11,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:11,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:13,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:13,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:16,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:19,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:19,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:19,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:19,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:19,839 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:20,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:20,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:22,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:24,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:24,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:26,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:26,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:26,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:29,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:29,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:29,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:29,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:29,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:30,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:30,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:32,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:34,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:34,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:36,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:38,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:09:39,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:39,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:39,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:39,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:39,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:40,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:40,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:41,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:41,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:42,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:44,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:45,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:46,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:50,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:50,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:50,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:50,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:50,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:50,600 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:51,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:52,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:56,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:56,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:56,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:56,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:58,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:00,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:01,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:10:01,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:10:01,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:10:01,198 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:10:02,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:10:02,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:02,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:05,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:10:05,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:10:05,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:10:05,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:10:06,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:10:06,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:07,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:08,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:08,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:10:11,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:11,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:13,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:18,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:23,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:26,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:26,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:28,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:33,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:38,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:10:39,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:42,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:42,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:45,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:50,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:55,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:57,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:57,279 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:59,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_ae28d8db362e0aa9f573.png with policy now +2023-07-12 00:10:59,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_616207d75a28a906c55b.png with policy now +2023-07-12 00:10:59,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_2c0d6dfc2d246283392e.png with policy now +2023-07-12 00:10:59,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_ed7f0d8809d791b958b4.png with policy now +2023-07-12 00:10:59,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:10:59,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:10:59,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:10:59,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:00,079 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e2ufjigs-media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 00:11:00,120 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6dvjs6kd-media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 00:11:00,121 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wsb7bsl6-media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 00:11:00,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:00,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 00:11:00,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 00:11:00,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 00:11:00,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 00:11:00,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:11:00,254 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sc1ybcfp-media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 00:11:00,524 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lnva8x1k-media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 00:11:00,538 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9qhfu4uq-media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 00:11:00,545 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q8h0813r-media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 00:11:00,644 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0l8safbx-media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 00:11:00,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:04,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:05,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:06,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:08,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:08,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:11:09,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:11,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:12,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:12,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:12,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:14,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:16,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:17,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:20,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:20,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:20,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:20,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:20,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:21,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:22,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:22,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:24,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:26,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:27,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:27,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:27,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:30,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:30,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:30,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:30,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:30,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:30,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:32,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:33,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:34,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:38,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:38,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:11:38,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:40,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:40,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:40,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:40,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:40,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:41,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:42,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:42,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:42,665 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:44,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:44,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:48,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:49,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:50,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:51,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:51,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:51,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:51,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:52,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:54,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:55,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:56,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:57,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:57,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:58,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:00,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:00,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:01,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:01,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:01,866 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:01,868 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:02,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:04,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:05,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:06,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:08,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:08,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:12:10,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:11,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:11,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:11,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:11,482 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:11,484 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:12,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:13,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:13,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:12:14,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:16,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:16,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:18,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:20,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:21,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:21,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:21,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:21,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:21,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:22,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:24,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:26,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:26,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:28,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:28,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:28,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:12:30,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:31,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:31,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:31,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:31,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:32,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:32,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:32,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:36,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:36,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:36,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:36,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:36,650 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:37,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:38,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:38,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:38,487 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:12:43,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:43,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:43,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:12:48,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:53,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:58,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:59,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:59,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:04,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:08,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:13:09,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:14,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:14,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:15,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:20,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:25,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:29,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:29,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:30,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:30,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_beffc069b9ded929df09.png with policy now +2023-07-12 00:13:30,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,882 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_83474bbbae95371e4fe1.png with policy now +2023-07-12 00:13:30,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_8ede4c40d45be37a3114.png with policy now +2023-07-12 00:13:30,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,903 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_cb5f34fc925cd4802446.png with policy now +2023-07-12 00:13:30,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:30,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:13:30,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:13:30,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:13:31,159 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fnjomyyl-media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 00:13:31,211 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/japiy359-media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 00:13:31,221 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zjf1c6ic-media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 00:13:31,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e57km2oq-media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 00:13:31,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:13:31,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 00:13:31,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 00:13:31,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 00:13:31,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 00:13:31,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:13:31,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nffzyvga-media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 00:13:31,895 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5w5ak3yy-media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 00:13:31,907 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e1yflci4-media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 00:13:31,981 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ns9mxiuz-media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 00:13:35,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:36,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:38,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:13:38,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:40,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:41,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:41,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:42,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:44,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:44,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:46,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:46,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:48,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:50,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:50,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:13:50,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:13:50,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:13:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:13:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:52,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:52,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:54,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:56,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:57,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:58,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:59,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:59,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:59,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:59,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:13:59,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:13:59,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:00,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:00,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:02,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:03,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:04,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:06,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:08,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:14:08,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:08,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:09,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:09,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:09,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:09,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:09,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:10,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:13,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:14,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:14:14,460 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:14:14,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:16,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:18,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:19,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:20,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:20,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:20,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:20,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:20,589 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:22,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:24,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:25,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:26,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:29,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:14:29,460 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:14:30,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:30,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:30,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:30,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:30,746 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:30,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:31,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:32,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:34,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:35,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:38,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:14:38,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:40,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:40,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:40,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:40,690 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:40,692 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:41,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:41,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:42,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:44,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:45,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:14:45,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:14:47,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:48,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:50,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:51,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:51,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:51,306 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:51,308 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:51,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:52,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:52,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:56,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:58,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:58,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:00,674 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:01,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:01,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:02,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:15:02,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:15:02,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:15:02,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:15:02,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:15:04,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:04,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:06,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:15:06,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:15:06,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:15:06,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:15:06,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:15:06,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:08,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:15:08,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:09,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:14,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:16,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:16,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:19,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:24,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:29,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:31,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:31,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:35,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:38,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:15:41,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:46,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:46,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:46,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:51,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:56,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:00,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_6c49012791de94816289.png with policy now +2023-07-12 00:16:00,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,676 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_5fc1a953a335a81d0538.png with policy now +2023-07-12 00:16:00,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_f43b6d21c133e2a6b003.png with policy now +2023-07-12 00:16:00,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_8bd9326976928e17b3ca.png with policy now +2023-07-12 00:16:00,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:00,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:00,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:00,738 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:00,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:00,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_6c49012791de94816289.png +2023-07-12 00:16:00,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 00:16:00,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 00:16:00,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 00:16:00,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:16:00,974 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/42ait1o9-media/images/validation_50_6c49012791de94816289.png +2023-07-12 00:16:01,003 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ef1s45rv-media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 00:16:01,006 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1pe31w2-media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 00:16:01,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mv7jnge0-media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 00:16:01,382 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9r8deach-media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 00:16:01,426 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9a0n3hwt-media/images/validation_50_6c49012791de94816289.png +2023-07-12 00:16:01,688 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rnmo2lgf-media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 00:16:01,766 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6q92dm9z-media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 00:16:01,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:01,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:02,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:06,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:08,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:08,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:16:08,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:10,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:11,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:12,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:13,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:14,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:16,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:17,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:17,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:18,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:20,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:20,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:20,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:20,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:20,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:20,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:22,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:24,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:24,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:28,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:29,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:30,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:31,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:31,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:31,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:31,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:31,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:32,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:33,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:33,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:34,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:36,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:38,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:16:38,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:40,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:40,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:40,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:40,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:40,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:40,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:41,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:42,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:44,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:45,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:46,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:48,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:48,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:50,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:50,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:51,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:51,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:51,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:51,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:51,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:52,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:54,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:56,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:56,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:00,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:00,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:00,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:00,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:00,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:00,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:02,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:02,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:03,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:03,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:04,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:07,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:08,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:17:08,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:10,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:10,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:10,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:10,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:10,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:10,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:12,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:13,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:14,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:16,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:18,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:18,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:18,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:20,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:20,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:20,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:20,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:20,996 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:21,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:22,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:23,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:24,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:28,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:29,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:30,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:31,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:31,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:31,490 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:31,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:31,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:32,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:33,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:33,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:35,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:36,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:36,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:36,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:36,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:37,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:37,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:38,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:17:39,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:40,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:45,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:48,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:48,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:50,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:55,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:00,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:03,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:03,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:06,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:08,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:18:12,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:17,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:18,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:18,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:22,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:27,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:30,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_6ae01c6a6a2bfa713e44.png with policy now +2023-07-12 00:18:30,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_cfabe222535005b7c73d.png with policy now +2023-07-12 00:18:30,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_ddf1c39281d80912060e.png with policy now +2023-07-12 00:18:30,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_ed363dbde584f9cd8a96.png with policy now +2023-07-12 00:18:30,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:30,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:18:30,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:18:30,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:18:31,007 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q5g873z9-media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 00:18:31,013 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6duhdnk9-media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 00:18:31,019 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ubg9twvg-media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 00:18:31,030 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/73miy2qg-media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 00:18:31,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:18:31,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 00:18:31,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 00:18:31,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 00:18:31,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 00:18:31,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:18:31,453 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/19aqlk2v-media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 00:18:31,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5x31q3in-media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 00:18:31,468 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8zx1d2g4-media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 00:18:31,620 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlt62v6a-media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 00:18:33,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:33,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:33,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:35,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:37,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:38,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:18:38,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:40,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:41,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:43,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:43,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:45,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:47,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:48,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:48,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:48,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:49,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:49,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:49,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:18:49,427 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:18:49,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:18:50,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:18:51,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:53,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:54,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:57,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:59,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:59,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:59,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:59,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:18:59,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:18:59,717 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:00,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:01,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:03,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:04,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:04,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:04,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:07,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:08,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:19:09,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:10,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:10,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:10,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:10,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:10,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:11,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:13,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:15,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:16,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:17,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:19,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:19,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:19,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:20,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:20,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:20,562 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:20,564 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:21,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:21,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:23,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:25,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:27,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:27,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:29,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:30,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:30,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:30,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:30,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:31,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:32,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:33,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:34,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:34,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:35,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:37,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:38,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:19:38,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:39,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:40,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:40,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:40,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:40,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:41,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:43,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:43,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:45,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:47,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:49,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:50,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:50,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:51,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:51,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:51,094 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:51,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:51,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:51,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:53,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:54,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:55,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:57,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:59,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:01,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:20:01,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:20:01,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:20:01,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:20:01,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:20:01,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:03,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:05,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:05,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:05,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:05,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:05,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:20:05,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:20:05,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:20:05,609 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:20:06,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:20:07,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:08,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:20:09,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:10,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:15,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:20,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:20,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:20,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:25,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:30,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:35,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:35,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:36,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:38,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:20:42,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:47,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:50,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:50,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:52,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:57,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:00,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_09c22363614ab958c3bc.png with policy now +2023-07-12 00:21:00,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_eba1f69e2b11ef8f44d3.png with policy now +2023-07-12 00:21:00,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_d01dfe2d2aeaa1bec479.png with policy now +2023-07-12 00:21:00,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,359 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_24254896e341a078f630.png with policy now +2023-07-12 00:21:00,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:00,404 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:00,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:00,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:00,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:00,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 00:21:00,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_24254896e341a078f630.png +2023-07-12 00:21:00,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 00:21:00,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 00:21:00,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:21:00,657 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9gqvcafo-media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 00:21:00,681 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ysk754mc-media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 00:21:00,702 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bjb6u0r3-media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 00:21:00,709 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/woh0j4tr-media/images/validation_70_24254896e341a078f630.png +2023-07-12 00:21:01,050 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qphexw0y-media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 00:21:01,085 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8q4mwhlm-media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 00:21:01,091 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y6jl2dns-media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 00:21:01,400 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kpm4w6i7-media/images/validation_70_24254896e341a078f630.png +2023-07-12 00:21:03,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:04,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:05,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:05,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:06,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:08,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:08,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:21:08,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:10,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:12,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:13,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:14,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:16,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:18,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:19,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:19,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:19,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:19,581 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:19,583 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:20,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:20,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:20,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:20,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:22,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:24,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:24,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:28,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:29,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:29,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:29,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:29,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:29,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:29,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:30,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:32,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:34,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:36,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:36,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:36,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:38,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:21:38,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:40,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:40,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:40,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:40,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:40,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:40,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:40,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:44,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:45,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:46,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:48,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:50,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:50,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:50,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:50,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:50,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:50,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:51,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:51,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:51,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:54,564 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:56,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:56,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:58,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:00,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:00,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:00,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:00,503 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:00,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:01,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:02,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:04,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:06,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:06,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:06,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:06,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:08,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:22:08,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:11,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:11,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:11,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:11,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:11,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:12,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:12,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:14,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:17,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:18,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:20,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:21,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:21,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:21,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:21,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:21,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:21,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:21,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:22,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:22,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:24,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:28,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:28,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:30,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:31,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:31,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:31,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:31,391 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:31,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:32,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:34,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:34,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:36,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:36,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:36,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:36,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:36,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:36,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:36,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:36,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:38,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:22:38,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:39,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:44,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:49,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:52,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:52,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:55,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:00,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:05,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:07,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:07,028 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:08,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:23:10,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:15,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:20,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:22,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:22,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:26,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:30,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_64b2c49cff2d363d20ad.png with policy now +2023-07-12 00:23:30,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_f3e5374d59497f9be535.png with policy now +2023-07-12 00:23:30,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_653670180f2536edf28e.png with policy now +2023-07-12 00:23:30,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_ca894d05d078201593b2.png with policy now +2023-07-12 00:23:30,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:30,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:23:30,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:23:30,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:23:30,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:23:30,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 00:23:30,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 00:23:30,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_653670180f2536edf28e.png +2023-07-12 00:23:30,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 00:23:30,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:23:30,888 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/odczyus8-media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 00:23:30,949 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ebgh994-media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 00:23:30,950 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zduxw7n-media/images/validation_80_653670180f2536edf28e.png +2023-07-12 00:23:30,978 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nrir2twy-media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 00:23:31,297 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/evgcmjsm-media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 00:23:31,299 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fnlh6iys-media/images/validation_80_653670180f2536edf28e.png +2023-07-12 00:23:31,376 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7yy9cotf-media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 00:23:31,627 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mceu715a-media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 00:23:31,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:34,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:36,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:36,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:37,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:37,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:38,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:23:38,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:40,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:40,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:42,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:44,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:46,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:47,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:48,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:49,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:49,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:23:49,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:23:49,942 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:23:50,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:23:50,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:52,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:52,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:52,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:53,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:54,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:58,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:58,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:59,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:59,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:23:59,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:23:59,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:23:59,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:00,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:02,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:04,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:06,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:07,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:07,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:08,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:24:08,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:09,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:09,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:09,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:09,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:09,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:09,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:10,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:12,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:15,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:16,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:18,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:20,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:20,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:20,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:20,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:20,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:20,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:20,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:22,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:22,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:24,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:25,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:28,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:29,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:29,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:29,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:29,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:30,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:30,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:30,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:34,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:36,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:36,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:37,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:37,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:38,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:24:38,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:40,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:40,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:40,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:40,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:41,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:41,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:42,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:44,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:46,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:47,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:50,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:51,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:51,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:51,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:51,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:51,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:52,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:52,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:53,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:53,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:54,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:58,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:58,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:01,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:02,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:25:02,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:25:02,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:25:02,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:25:03,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:25:03,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:04,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:05,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:06,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:25:06,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:25:06,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:25:06,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:25:07,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:25:07,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:08,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:25:08,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:08,781 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:09,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:09,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:14,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:19,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:23,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:23,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:24,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:29,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:35,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:38,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:25:38,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:38,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:40,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:45,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:50,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:53,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:53,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:55,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:00,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,507 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_d4b9dc3d41c12fcb9fe7.png with policy now +2023-07-12 00:26:00,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_21a56bd8f7d955ce07a3.png with policy now +2023-07-12 00:26:00,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_8b813df4910f6e14fe9f.png with policy now +2023-07-12 00:26:00,551 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_b276db68b1c2650b37f5.png with policy now +2023-07-12 00:26:00,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:00,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:00,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:00,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:00,809 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s9933rxu-media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 00:26:00,870 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p6brayzl-media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 00:26:00,918 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7t84wvff-media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 00:26:00,922 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dvwh5op3-media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 00:26:01,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:01,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 00:26:01,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 00:26:01,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 00:26:01,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 00:26:01,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:26:01,481 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ov1qe56-media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 00:26:01,482 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bx2foth0-media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 00:26:01,485 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o0rzi9o4-media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 00:26:01,504 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ml0som78-media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 00:26:01,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:05,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:07,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:07,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:08,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:26:09,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:09,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:11,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:11,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:13,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:13,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:15,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:17,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:18,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:19,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:19,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:19,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:19,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:19,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:20,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:21,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:23,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:24,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:24,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:24,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:25,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:29,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:29,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:29,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:29,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:29,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:29,453 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:30,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:31,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:33,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:34,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:37,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:38,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:26:39,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:39,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:39,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:39,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:40,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:40,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:40,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:40,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:41,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:41,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:43,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:44,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:47,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:49,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:49,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:49,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:49,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:49,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:50,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:50,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:51,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:53,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:54,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:54,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:55,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:57,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:59,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:59,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:59,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:59,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:59,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:00,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:00,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:01,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:03,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:06,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:07,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:08,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:27:09,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:09,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:09,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:27:10,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:10,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:10,086 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:10,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:10,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:11,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:12,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:13,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:17,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:17,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:19,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:20,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:20,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:20,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:20,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:21,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:21,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:22,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:25,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:25,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:27:25,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:27,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:28,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:29,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:31,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:31,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:31,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:31,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:31,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:33,384 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:33,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:35,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:35,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:35,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:35,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:35,670 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:36,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:37,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:38,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:27:38,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:40,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:40,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:27:44,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:49,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:54,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:55,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:55,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:00,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:05,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:08,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:28:10,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:10,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:10,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:15,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:20,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:25,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:25,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:25,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:29,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_661b1a72c771caa0e001.png with policy now +2023-07-12 00:28:29,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_85a79c24055b252b391d.png with policy now +2023-07-12 00:28:29,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_e3b8328c3efd11dc0868.png with policy now +2023-07-12 00:28:29,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_5f675bc00f07f97610c2.png with policy now +2023-07-12 00:28:30,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:30,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:28:30,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:28:30,023 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:28:30,255 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bp294jcq-media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 00:28:30,286 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ntvbmgn2-media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 00:28:30,315 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nqyst2kx-media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 00:28:30,325 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bad3jhhc-media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 00:28:30,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:28:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 00:28:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 00:28:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 00:28:30,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 00:28:30,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:28:30,860 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/58kmyqby-media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 00:28:30,866 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iuwy4cyu-media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 00:28:30,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oyf8n66j-media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 00:28:30,877 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/15hkeu2g-media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 00:28:31,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:33,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:36,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:37,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:38,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:28:39,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:40,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:40,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:40,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:41,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:41,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:43,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:47,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:47,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:49,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:49,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:49,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:28:49,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:28:49,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:28:50,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:28:51,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:52,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:53,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:55,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:55,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:55,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:57,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:58,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:59,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:59,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:28:59,180 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:28:59,182 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:28:59,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:01,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:03,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:04,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:05,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:07,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:08,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:29:09,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:09,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:09,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:09,778 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:09,780 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:10,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:10,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:10,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:11,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:13,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:14,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:15,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:19,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:20,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:20,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:20,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:20,147 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:20,149 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:20,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:21,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:25,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:25,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:25,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:25,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:27,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:29,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:30,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:31,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:31,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:31,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:31,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:31,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:33,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:35,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:35,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:37,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:38,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:29:39,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:40,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:40,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:40,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:41,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:41,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:41,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:41,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:41,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:43,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:45,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:45,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:47,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:49,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:51,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:51,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:51,488 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:51,488 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:51,490 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:51,752 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:53,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:55,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:55,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:55,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:57,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:57,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:59,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:01,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:30:01,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:30:01,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:30:01,053 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:30:01,780 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:30:03,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:03,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:05,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:30:05,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:30:05,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:30:05,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:30:05,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:30:05,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:07,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:08,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:08,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:30:10,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:10,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:30:13,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:18,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:23,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:25,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:25,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:30:29,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:34,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:38,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:30:40,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:40,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:40,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:30:46,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:51,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:56,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:56,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:56,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:00,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,647 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_ed89fce5f94211880d08.png with policy now +2023-07-12 00:31:00,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_519be1aabb2ce5adb552.png with policy now +2023-07-12 00:31:00,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_0a38067c3b4b0763814d.png with policy now +2023-07-12 00:31:00,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_57b921c4856dfbee5311.png with policy now +2023-07-12 00:31:00,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:00,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:00,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:00,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 00:31:00,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 00:31:00,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:31:01,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3uw9s60a-media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 00:31:01,050 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/671lna2s-media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 00:31:01,057 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oiczommb-media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 00:31:01,118 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i7iaks7w-media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 00:31:01,456 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q8dchois-media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 00:31:01,641 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1gphfzhd-media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 00:31:01,673 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cghivl3s-media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 00:31:01,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:01,859 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/96lzp3r5-media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 00:31:05,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:07,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:07,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:08,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:31:11,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:11,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:11,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:11,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:13,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:13,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:15,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:17,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:19,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:21,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:21,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:21,739 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:21,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:21,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:21,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:23,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:25,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:25,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:26,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:26,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:27,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:29,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:30,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:31,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:31,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:31,028 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:31,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:31,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:31,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:35,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:36,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:37,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:38,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:31:40,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:41,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:41,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:41,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:41,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:41,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:41,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:41,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:42,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:44,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:46,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:46,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:48,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:51,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:51,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:51,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:51,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:51,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:52,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:52,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:54,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:56,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:56,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:56,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:57,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:58,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:02,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:02,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:02,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:02,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:02,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:02,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:03,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:04,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:06,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:08,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:08,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:32:08,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:11,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:11,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:11,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:11,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:11,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:11,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:12,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:12,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:13,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:14,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:16,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:18,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:19,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:22,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:22,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:22,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:22,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:22,212 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:23,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:24,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:25,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:26,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:26,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:26,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:30,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:30,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:32,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:32,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:32,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:32,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:32,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:33,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:34,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:36,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:36,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:37,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:37,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:37,309 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:37,311 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:38,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:38,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:32:40,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:41,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:41,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:41,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:46,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:51,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:56,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:56,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:56,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:01,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:06,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:08,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:33:12,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:12,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:12,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:17,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:22,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:27,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:27,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:27,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:31,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_b4dbdc5959d71b1b0010.png with policy now +2023-07-12 00:33:31,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_3aa26b32048f4df01e9f.png with policy now +2023-07-12 00:33:31,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_18e3cfcb4e5db9c7981b.png with policy now +2023-07-12 00:33:31,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_f3f35ebd04d7e55194ae.png with policy now +2023-07-12 00:33:31,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:33:31,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:33:31,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:33:31,944 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:33:32,262 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jntiff99-media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 00:33:32,303 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cr02ki2y-media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 00:33:32,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:33:32,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 00:33:32,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 00:33:32,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 00:33:32,315 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4f9piivd-media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 00:33:32,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 00:33:32,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:33:32,327 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vjyn8i9f-media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 00:33:32,839 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6e9gix8a-media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 00:33:32,922 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l167ysr6-media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 00:33:32,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:32,954 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bqknbxsp-media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 00:33:33,043 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cgq95ug3-media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 00:33:36,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:38,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:38,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:33:38,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:40,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:41,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:33:42,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:42,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:42,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:43,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:46,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:48,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:49,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:50,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:50,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:33:50,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:33:50,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:33:50,959 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:33:51,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:33:52,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:54,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:56,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:57,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:57,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:58,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:59,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:00,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:00,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:00,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:00,042 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:00,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:00,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:02,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:04,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:05,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:06,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:08,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:08,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:34:09,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:09,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:09,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:09,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:10,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:11,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:12,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:12,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:12,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:14,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:16,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:16,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:18,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:19,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:19,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:19,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:19,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:20,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:21,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:22,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:24,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:26,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:27,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:27,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:30,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:30,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:30,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:30,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:30,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:31,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:32,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:32,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:34,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:38,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:38,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:38,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:34:40,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:40,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:40,777 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:40,777 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:40,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:41,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:42,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:43,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:43,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:43,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:46,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:48,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:48,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:50,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:51,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:51,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:51,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:51,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:52,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:54,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:54,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:56,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:58,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:58,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:58,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:59,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:00,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:02,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:35:02,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:35:02,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:35:02,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:35:02,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:35:04,582 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:04,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:06,589 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:06,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:35:06,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:35:06,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:35:06,844 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:35:07,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:35:08,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:35:08,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:10,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:13,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:13,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:35:15,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:20,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:25,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:28,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:28,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:35:30,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:35,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:38,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:35:41,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:43,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:43,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:35:46,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:51,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:56,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:58,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:58,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:01,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,241 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_3e5c971e8f903ffb6755.png with policy now +2023-07-12 00:36:01,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_b27cf42472a6e5022d7a.png with policy now +2023-07-12 00:36:01,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_db9b8ff059ac75e60033.png with policy now +2023-07-12 00:36:01,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_1846f2582dc6add0423d.png with policy now +2023-07-12 00:36:01,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:01,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:01,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:01,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:01,584 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0inophzk-media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 00:36:01,687 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dkai2j6d-media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 00:36:01,703 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iveoso0e-media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 00:36:01,704 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvceg8e5-media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 00:36:01,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:01,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 00:36:01,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 00:36:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 00:36:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 00:36:01,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:36:02,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:02,412 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xz0731oq-media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 00:36:02,413 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d04m3zfu-media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 00:36:02,416 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sljtltlp-media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 00:36:02,456 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vl09kwjy-media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 00:36:06,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:08,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:08,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:36:08,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:10,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:10,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:12,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:13,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:13,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:13,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:14,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:18,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:19,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:20,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:20,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:20,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:20,243 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:20,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:20,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:22,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:24,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:24,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:26,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:29,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:29,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:29,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:30,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:30,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:30,102 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:30,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:30,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:30,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:32,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:34,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:34,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:36,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:38,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:36:38,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:40,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:40,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:40,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:40,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:40,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:40,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:42,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:44,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:44,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:44,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:46,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:46,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:50,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:50,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:50,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:50,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:50,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:50,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:51,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:52,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:54,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:56,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:57,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:59,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:59,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:00,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:00,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:00,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:00,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:00,898 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:00,898 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:02,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:02,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:04,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:07,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:08,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:37:08,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:10,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:11,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:11,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:11,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:11,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:11,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:12,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:13,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:14,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:14,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:16,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:18,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:18,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:20,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:21,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:21,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:21,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:21,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:21,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:22,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:23,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:26,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:28,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:28,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:29,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:29,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:30,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:31,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:31,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:31,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:31,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:31,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:34,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:35,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:36,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:36,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:36,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:37,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:37,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:38,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:37:39,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:39,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:44,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:44,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:44,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:49,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:54,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:59,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:59,625 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:59,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:04,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:08,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:38:10,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:14,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:38:14,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:38:15,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:20,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:25,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:29,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:38:29,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:38:30,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_20276ffa7780c15c8826.png with policy now +2023-07-12 00:38:30,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_5f10192137f91e3d88e7.png with policy now +2023-07-12 00:38:30,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_f11466fe48b33d6f9da9.png with policy now +2023-07-12 00:38:30,675 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_8cf36f31bd00d9abfd34.png with policy now +2023-07-12 00:38:30,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:38:30,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:38:30,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:38:30,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:30,709 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:38:30,943 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m5yivnxi-media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 00:38:30,987 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/erf6mck5-media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 00:38:31,001 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3obh3r2x-media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 00:38:31,003 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/di5yinl7-media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 00:38:31,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:38:31,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 00:38:31,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 00:38:31,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 00:38:31,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 00:38:31,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:38:31,658 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xgahyfh8-media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 00:38:31,716 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z0ip3jvj-media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 00:38:31,721 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ialsxms9-media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 00:38:31,744 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gcqqedml-media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 00:38:35,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:36,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:38,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:38,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:38:40,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:40,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:38:41,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:42,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:44,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:45,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:38:45,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:38:46,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:47,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:50,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:50,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:38:50,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:38:50,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:38:50,552 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:38:51,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:38:52,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:52,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:54,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:56,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:58,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:00,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:00,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:00,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:00,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:00,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:00,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:00,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:01,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:02,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:03,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:04,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:06,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:08,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:39:08,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:10,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:10,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:10,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:10,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:10,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:11,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:12,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:13,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:14,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:15,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:15,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:16,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:19,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:20,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:21,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:21,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:21,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:21,117 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:21,315 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:22,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:24,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:24,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:28,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:30,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:30,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:31,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:31,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:31,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:31,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:31,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:31,956 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:32,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:34,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:35,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:36,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:38,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:38,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:39:40,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:40,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:41,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:41,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:41,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:41,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:42,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:42,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:45,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:46,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:46,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:46,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:48,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:51,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:52,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:52,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:52,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:52,604 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:52,606 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:53,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:54,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:56,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:57,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:58,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:02,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:02,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:02,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:02,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:02,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:40:02,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:40:02,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:40:02,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:40:03,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:40:04,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:06,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:07,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:07,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:40:07,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:40:07,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:40:07,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:40:07,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:40:08,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:08,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:40:10,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:12,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:17,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:17,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:18,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:23,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:28,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:32,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:32,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:34,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:38,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:40:39,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:44,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:47,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:47,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:50,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:55,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:00,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:01,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,804 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_ad78c61f0f7db6d206fd.png with policy now +2023-07-12 00:41:01,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_da52d8afd9d0f2869f66.png with policy now +2023-07-12 00:41:01,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,846 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_c4f6c02b99e6533a2eb2.png with policy now +2023-07-12 00:41:01,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_843770c3f4c58f7b4f6c.png with policy now +2023-07-12 00:41:01,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:01,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:01,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:01,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:02,113 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r15h5x44-media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 00:41:02,155 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wb8y7n3r-media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 00:41:02,164 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tlw86rco-media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 00:41:02,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:02,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:02,199 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xs3lziqw-media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 00:41:02,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:02,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 00:41:02,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 00:41:02,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 00:41:02,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 00:41:02,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:41:03,034 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3qlhbqq-media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 00:41:03,042 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yo57wtq9-media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 00:41:03,048 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hrwqwzb6-media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 00:41:03,105 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tll9zjp9-media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 00:41:05,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:06,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:08,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:41:08,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:10,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:11,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:11,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:12,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:16,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:17,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:17,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:18,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:20,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:21,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:21,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:21,306 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:21,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:21,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:22,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:22,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:24,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:26,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:27,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:30,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:31,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:31,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:31,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:31,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:31,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:32,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:32,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:32,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:32,892 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:34,731 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:36,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:38,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:38,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:41:40,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:41,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:41,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:41,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:41,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:41,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:42,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:43,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:44,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:47,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:47,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:48,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:48,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:50,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:51,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:51,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:51,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:51,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:51,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:52,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:53,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:54,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:56,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:58,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:00,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:01,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:01,466 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:01,466 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:01,468 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:01,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:02,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:03,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:03,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:04,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:04,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:08,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:42:08,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:09,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:10,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:12,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:12,031 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:12,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:12,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:12,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:12,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:14,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:16,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:18,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:18,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:18,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:19,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:20,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:21,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:21,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:21,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:21,981 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:22,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:22,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:24,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:26,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:28,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:30,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:30,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:32,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:32,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:32,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:32,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:32,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:33,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:33,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:34,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:35,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:36,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:37,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:37,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:37,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:37,534 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:37,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:38,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:42:38,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:40,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:40,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:45,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:49,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:49,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:51,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:56,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:01,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:04,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:04,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:06,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:08,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:43:11,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:16,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:19,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:19,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:22,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:27,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:31,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_897ca9ce40094218e736.png with policy now +2023-07-12 00:43:31,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_a6aefafa2d783d4803ef.png with policy now +2023-07-12 00:43:31,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_c063d5e9ec2283745c8b.png with policy now +2023-07-12 00:43:31,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_542fde61a9a7764e7963.png with policy now +2023-07-12 00:43:32,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:43:32,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:43:32,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:43:32,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:43:32,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:43:32,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 00:43:32,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 00:43:32,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 00:43:32,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 00:43:32,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:43:32,294 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a0lih3sw-media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 00:43:32,347 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5d0vwiv3-media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 00:43:32,402 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ecuuvjg-media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 00:43:32,449 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/drzfcr5i-media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 00:43:33,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:33,203 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mef4hn2q-media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 00:43:33,400 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3p30hdae-media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 00:43:33,693 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3nl0jkiu-media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 00:43:33,905 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jv0cgo68-media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 00:43:34,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:34,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:37,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:38,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:38,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:43:39,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:41,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:41,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:43:43,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:43,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:47,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:49,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:49,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:49,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:49,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:51,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:43:51,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:43:51,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:43:51,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:43:51,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:43:51,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:53,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:54,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:55,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:57,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:59,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:00,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:00,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:00,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:00,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:00,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:01,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:03,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:04,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:04,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:05,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:05,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:07,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:08,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:44:09,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:10,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:10,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:10,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:10,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:10,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:11,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:11,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:15,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:15,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:17,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:19,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:19,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:19,845 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:20,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:20,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:20,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:20,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:20,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:21,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:23,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:25,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:25,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:27,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:29,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:30,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:30,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:30,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:30,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:30,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:31,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:33,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:34,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:34,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:35,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:36,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:37,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:38,568 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:44:40,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:40,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:40,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:40,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:41,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:41,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:41,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:43,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:47,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:47,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:49,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:49,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:49,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:51,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:52,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:52,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:52,067 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:52,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:52,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:53,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:53,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:57,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:58,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:59,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:01,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:02,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:45:02,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:45:02,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:45:02,419 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:45:03,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:45:03,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:04,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:04,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:05,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:06,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:45:06,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:45:06,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:45:06,853 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:45:07,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:45:07,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:08,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:08,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:45:09,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:13,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:18,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:19,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:19,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:23,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:28,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:33,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:35,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:35,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:38,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:45:39,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:44,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:49,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:50,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:50,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:55,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:00,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:01,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_7e7561227c49152fdf02.png with policy now +2023-07-12 00:46:01,335 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,335 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_249e7863a11a9d2249ff.png with policy now +2023-07-12 00:46:01,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_582ff9cdb53e3daa2736.png with policy now +2023-07-12 00:46:01,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_80a5f2920f4985e1b760.png with policy now +2023-07-12 00:46:01,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:01,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:01,400 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:01,402 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:01,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:01,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 00:46:01,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 00:46:01,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 00:46:01,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 00:46:01,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:46:01,735 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ko8estw4-media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 00:46:02,044 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6v5nktoq-media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 00:46:02,154 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tgefjzy3-media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 00:46:02,273 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rafxeil0-media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 00:46:02,367 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r7o0qrel-media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 00:46:02,701 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nvwtn1vs-media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 00:46:02,716 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m7dz5om7-media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 00:46:03,068 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2adf7zp-media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 00:46:05,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:05,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:05,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:06,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:08,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:46:09,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:11,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:11,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:12,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:13,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:15,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:17,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:17,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:20,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:20,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:20,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:20,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:20,890 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:20,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:21,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:21,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:22,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:23,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:25,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:27,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:28,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:29,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:30,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:30,045 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:30,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:30,047 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:30,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:31,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:34,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:35,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:35,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:35,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:37,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:38,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:46:39,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:40,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:40,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:40,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:40,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:40,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:40,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:41,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:45,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:46,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:47,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:49,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:50,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:50,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:51,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:51,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:51,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:51,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:51,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:52,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:53,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:55,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:57,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:57,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:01,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:01,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:01,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:01,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:01,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:01,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:02,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:03,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:05,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:06,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:06,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:07,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:08,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:47:09,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:11,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:11,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:11,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:11,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:11,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:12,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:12,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:13,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:17,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:18,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:19,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:21,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:22,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:22,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:22,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:22,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:22,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:22,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:23,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:23,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:25,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:27,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:28,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:29,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:31,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:33,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:33,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:33,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:33,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:33,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:34,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:35,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:37,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:37,360 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:37,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:37,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:37,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:37,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:37,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:37,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:38,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:47:39,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:39,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:44,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:49,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:52,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:52,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:55,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:00,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:05,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:07,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:07,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:08,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:48:10,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:15,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:20,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:22,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:22,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:26,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:31,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:32,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_bcd830ba6ea2f7b921a8.png with policy now +2023-07-12 00:48:32,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,350 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_9f041d8a799e7d60ca1c.png with policy now +2023-07-12 00:48:32,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_2942fa312a4899f548b2.png with policy now +2023-07-12 00:48:32,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_57486bf611e806418a3a.png with policy now +2023-07-12 00:48:32,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:48:32,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:48:32,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:48:32,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:48:32,652 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/djabuexf-media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 00:48:32,665 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5qx4mf79-media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 00:48:32,684 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6bgxbbca-media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 00:48:32,705 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/acv6gxw6-media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 00:48:33,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:48:33,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 00:48:33,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 00:48:33,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 00:48:33,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 00:48:33,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:48:33,416 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qpoq2xxr-media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 00:48:33,425 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/84bbko12-media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 00:48:33,440 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/he91l0op-media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 00:48:33,440 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yg0qufqz-media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 00:48:37,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:37,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:37,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:38,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:38,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:48:40,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:42,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:42,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:48:42,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:44,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:46,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:47,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:48,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:51,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:48:51,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:48:51,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:48:51,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:48:52,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:48:52,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:52,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:52,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:53,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:54,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:56,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:58,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:58,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:01,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:01,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:01,776 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:01,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:02,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:02,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:03,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:04,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:07,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:07,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:07,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:08,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:49:09,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:09,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:11,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:13,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:13,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:13,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:13,026 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:13,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:15,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:15,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:17,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:19,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:20,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:21,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:22,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:22,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:23,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:23,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:23,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:23,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:24,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:25,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:25,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:27,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:29,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:30,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:31,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:32,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:32,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:32,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:32,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:33,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:33,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:36,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:37,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:37,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:37,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:38,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:49:39,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:41,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:41,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:42,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:42,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:42,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:42,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:43,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:43,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:47,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:47,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:49,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:52,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:52,879 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:52,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:53,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:53,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:53,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:53,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:53,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:54,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:55,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:57,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:58,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:59,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:03,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:04,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:50:04,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:04,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:50:04,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:50:04,043 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:50:04,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:50:05,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:08,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:08,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:08,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:50:09,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:50:09,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:09,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:50:09,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:50:09,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:50:09,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:50:09,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:11,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:14,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:19,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:23,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:23,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:25,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:30,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:35,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:38,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:50:38,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:38,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:40,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:45,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:50,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:53,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:53,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:56,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:01,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:03,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_1689c770ac115a929d63.png with policy now +2023-07-12 00:51:03,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,973 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_18d5cba2a1158df428aa.png with policy now +2023-07-12 00:51:03,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_c50d65d1788fc664335f.png with policy now +2023-07-12 00:51:03,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_2b5b9594c7937018b6be.png with policy now +2023-07-12 00:51:04,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:04,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:04,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:04,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:04,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:04,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 00:51:04,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 00:51:04,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 00:51:04,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 00:51:04,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:51:07,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:07,897 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a784hyj8-media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 00:51:08,118 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pafr7q4n-media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 00:51:08,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:51:08,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:08,895 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:09,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:10,301 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/21dbln40-media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 00:51:11,416 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qj2xzx8f-media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 00:51:11,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:12,419 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c99dzjed-media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 00:51:12,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:12,660 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v54t3t3r-media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 00:51:13,136 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xx1mc0ih-media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 00:51:13,156 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rmdavyr6-media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 00:51:13,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:13,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:15,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:17,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:18,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:21,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:23,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:23,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:23,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:23,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:23,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:23,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:23,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:23,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:24,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:25,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:27,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:29,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:29,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:31,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:33,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:33,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:33,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:33,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:33,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:35,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:35,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:37,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:38,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:51:38,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:38,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:39,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:40,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:43,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:43,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:43,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:43,829 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:43,831 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:44,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:45,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:46,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:47,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:49,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:52,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:53,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:53,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:53,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:53,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:53,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:53,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:53,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:53,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:55,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:57,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:57,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:59,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:03,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:03,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:04,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:04,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:04,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:04,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:04,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:05,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:07,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:08,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:52:08,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:09,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:09,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:11,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:13,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:14,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:14,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:14,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:14,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:14,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:14,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:15,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:17,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:19,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:21,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:23,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:24,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:24,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:24,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:24,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:24,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:24,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:24,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:24,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:25,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:27,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:29,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:31,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:33,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:34,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:34,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:34,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:34,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:34,816 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:34,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:35,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:38,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:52:39,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:39,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:39,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:39,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:39,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:39,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:39,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:39,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:39,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:41,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:45,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:50,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:54,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:54,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:56,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:01,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:06,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:08,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:53:09,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:09,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:11,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:16,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:21,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:24,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:24,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:27,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:32,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:33,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:33,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_50004396bfec8e96747d.png with policy now +2023-07-12 00:53:33,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:33,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_a321ae427f3731921032.png with policy now +2023-07-12 00:53:34,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:34,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_009b86ce19fa976b3f5d.png with policy now +2023-07-12 00:53:34,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:34,018 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_fe7b08ac2a5afbc11b18.png with policy now +2023-07-12 00:53:34,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:53:34,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:53:34,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:53:34,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_a321ae427f3731921032.png +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 00:53:34,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 00:53:34,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:53:34,275 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m3j85j9c-media/images/validation_200_a321ae427f3731921032.png +2023-07-12 00:53:34,282 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fg2w37d7-media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 00:53:34,336 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jipjvvr4-media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 00:53:34,348 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c95uy6ly-media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 00:53:34,642 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/826u4wm8-media/images/validation_200_a321ae427f3731921032.png +2023-07-12 00:53:34,693 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tmx3q6kx-media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 00:53:35,173 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1pgxccs-media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 00:53:35,214 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qfjfp81t-media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 00:53:37,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:38,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:53:39,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:39,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:40,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:42,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:42,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:44,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:45,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:53:48,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:48,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:50,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:52,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:53,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:54,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:54,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:53:54,404 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:53:54,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:53:54,407 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:53:54,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:54,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:55,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:53:56,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:58,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:58,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:02,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:04,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:04,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:04,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:04,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:04,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:04,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:05,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:06,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:08,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:08,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:54:09,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:09,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:09,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:12,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:14,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:15,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:15,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:15,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:15,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:15,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:16,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:18,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:20,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:20,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:22,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:24,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:24,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:24,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:24,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:24,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:24,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:25,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:26,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:26,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:28,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:30,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:32,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:34,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:34,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:34,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:34,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:34,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:34,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:36,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:37,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:38,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:38,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:54:39,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:39,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:42,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:42,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:44,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:44,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:44,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:44,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:44,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:45,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:46,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:48,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:50,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:52,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:53,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:54,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:54,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:54,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:55,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:55,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:55,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:55,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:56,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:58,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:59,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:00,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:02,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:04,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:04,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:05,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:55:05,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:55:05,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:55:05,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:55:05,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:55:08,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:08,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:55:09,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:55:09,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:55:09,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:55:09,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:55:09,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:09,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:10,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:55:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:12,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:15,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:20,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:24,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:24,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:25,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:30,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:35,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:38,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:55:39,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:39,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:41,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:46,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:51,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:54,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:54,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:56,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:01,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:04,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_1f72d7f053af939e5276.png with policy now +2023-07-12 00:56:04,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_ade80344c97e522275e6.png with policy now +2023-07-12 00:56:04,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_b7889716d369f8c47db2.png with policy now +2023-07-12 00:56:04,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_30fb41bcb1ce089cb7f8.png with policy now +2023-07-12 00:56:04,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:04,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:04,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:04,123 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:04,374 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cfc04n82-media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 00:56:04,391 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p23aesx3-media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 00:56:04,411 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/511nam4n-media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 00:56:04,418 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cqm5wftm-media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 00:56:04,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:04,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 00:56:04,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 00:56:04,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 00:56:04,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 00:56:04,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:56:05,001 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jmn7z23o-media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 00:56:05,010 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pn608moc-media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 00:56:05,041 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2boiplso-media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 00:56:05,049 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qgcjrdo7-media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 00:56:07,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:08,596 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:56:08,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:09,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:09,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:12,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:12,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:13,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:14,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:16,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:18,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:18,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:20,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:22,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:23,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:23,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:23,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:23,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:23,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:23,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:24,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:24,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:26,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:28,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:30,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:32,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:32,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:32,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:32,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:32,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:32,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:34,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:34,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:36,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:38,597 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:56:39,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:39,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:39,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:40,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:42,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:42,466 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:42,466 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:42,468 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:42,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:42,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:44,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:45,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:46,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:50,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:51,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:52,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:52,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:52,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:52,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:52,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:52,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:54,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:54,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:54,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:56,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:56,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:00,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:01,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:02,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:03,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:03,106 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:03,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:03,108 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:03,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:04,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:06,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:07,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:08,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:57:09,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:10,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:10,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:11,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:12,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:13,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:13,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:13,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:13,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:13,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:13,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:15,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:18,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:20,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:22,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:24,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:24,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:24,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:24,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:24,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:25,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:25,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:25,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:26,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:28,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:29,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:30,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:34,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:34,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:34,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:34,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:34,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:34,772 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:35,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:36,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:38,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:38,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:57:39,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:39,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:39,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:39,431 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:39,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:40,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:40,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:40,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:40,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:42,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:45,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:50,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:55,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:55,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:55,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:00,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:05,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:08,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:58:10,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:10,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:10,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:15,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:20,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:25,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:25,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:26,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:31,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:33,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_f581c132868ccb4db30b.png with policy now +2023-07-12 00:58:33,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_1b6340fa1d30a762feb0.png with policy now +2023-07-12 00:58:33,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_58119209c10f17b2d40d.png with policy now +2023-07-12 00:58:33,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_7e556c2aaa57f2ce8afe.png with policy now +2023-07-12 00:58:33,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:58:33,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:58:33,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:58:33,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:58:34,160 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wtrufn3n-media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 00:58:34,192 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c3rfipes-media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 00:58:34,214 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/otfrs8eb-media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 00:58:34,225 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ewqgymju-media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 00:58:34,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:58:34,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 00:58:34,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 00:58:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 00:58:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 00:58:34,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:58:34,605 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c7caswnk-media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 00:58:34,607 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ymyiv7h1-media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 00:58:34,663 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/233hh82s-media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 00:58:34,665 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3a7781aq-media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 00:58:36,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:38,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:38,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:58:40,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:40,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:42,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:42,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:44,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:44,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:58:46,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:47,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:48,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:50,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:52,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:52,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:54,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:58:54,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:58:54,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:58:54,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:58:54,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:58:55,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:55,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:56,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:58,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:58,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:00,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:02,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:03,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:03,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:03,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:03,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:03,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:04,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:06,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:08,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:08,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:59:09,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:10,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:11,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:11,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:12,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:14,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:14,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:14,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:14,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:14,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:15,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:16,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:18,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:19,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:22,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:24,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:24,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:24,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:24,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:24,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:24,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:25,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:26,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:26,283 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:26,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:28,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:30,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:31,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:34,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:34,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:34,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:34,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:34,963 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:35,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:36,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:36,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:38,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:38,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:59:42,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:42,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:42,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:42,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:44,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:46,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:46,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:46,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:46,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:46,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:48,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:48,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:50,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:52,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:53,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:54,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:56,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:56,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:56,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:56,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:56,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:58,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:58,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:58,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:58,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:00,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:02,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:04,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:04,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:05,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:00:05,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:00:05,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:00:05,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:00:06,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:00:06,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:08,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:00:09,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:10,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:00:10,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:00:10,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:00:10,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:00:10,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:00:10,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:12,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:13,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:13,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:00:15,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:20,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:25,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:28,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:28,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:00:30,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:35,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:38,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:00:40,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:43,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:43,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:00:46,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:51,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:56,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:58,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:58,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:01,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:04,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_a1e3c8f3087f5e3012a5.png with policy now +2023-07-12 01:01:04,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_16d4939d62b11379b933.png with policy now +2023-07-12 01:01:04,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_9dcda9052cdd403f3897.png with policy now +2023-07-12 01:01:04,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,677 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_d631c6d270c38f7631a9.png with policy now +2023-07-12 01:01:04,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:04,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:04,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:04,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:04,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:04,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 01:01:04,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 01:01:04,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 01:01:04,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 01:01:04,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:01:04,970 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5ke4ir5m-media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 01:01:04,976 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dn4s0e9a-media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 01:01:04,981 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l9w9k5cf-media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 01:01:04,988 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/un9yskt0-media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 01:01:05,317 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/getpikyf-media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 01:01:05,342 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c2t95zfu-media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 01:01:05,355 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vrslw0cl-media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 01:01:05,665 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p9sewsmo-media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 01:01:06,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:08,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:01:08,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:12,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:12,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:13,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:13,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:14,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:14,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:16,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:17,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:18,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:20,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:22,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:24,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:24,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:24,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:24,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:24,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:24,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:26,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:28,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:28,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:28,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:28,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:30,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:32,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:33,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:33,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:33,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:33,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:33,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:34,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:34,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:38,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:01:38,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:40,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:40,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:42,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:44,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:44,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:44,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:44,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:44,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:44,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:44,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:46,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:47,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:49,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:51,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:51,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:55,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:55,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:55,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:55,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:55,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:55,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:57,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:57,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:59,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:59,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:59,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:02,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:03,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:04,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:04,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:04,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:04,842 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:05,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:05,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:07,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:08,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:08,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:02:09,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:13,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:14,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:14,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:14,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:14,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:14,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:14,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:14,948 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:15,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:15,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:17,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:19,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:19,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:23,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:24,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:25,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:25,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:25,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:25,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:25,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:25,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:27,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:29,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:29,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:29,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:29,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:31,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:35,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:35,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:35,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:35,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:35,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:35,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:35,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:37,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:38,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:02:40,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:40,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:40,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:40,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:40,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:41,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:41,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:43,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:44,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:44,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:45,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:50,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:55,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:59,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:59,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:00,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:05,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:08,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:03:11,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:14,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:03:14,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:16,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:21,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:26,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:29,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:03:29,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:31,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:34,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,731 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_5cfbeaed31a9337bdbad.png with policy now +2023-07-12 01:03:34,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,739 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_18282123a377a69416e7.png with policy now +2023-07-12 01:03:34,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,750 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_98861fa9aa94e0bb9466.png with policy now +2023-07-12 01:03:34,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_607b3eebed55e6abe830.png with policy now +2023-07-12 01:03:34,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:03:34,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:03:34,821 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:03:34,823 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:03:35,038 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pyasz1hw-media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 01:03:35,054 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fu7wmo5h-media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 01:03:35,065 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9m3pmamy-media/images/validation_240_18282123a377a69416e7.png +2023-07-12 01:03:35,096 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/orvb2iuf-media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 01:03:35,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:03:35,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 01:03:35,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 01:03:35,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_18282123a377a69416e7.png +2023-07-12 01:03:35,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 01:03:35,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:03:35,761 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z9om3u0x-media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 01:03:35,782 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/swo10992-media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 01:03:35,823 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ak8jej10-media/images/validation_240_18282123a377a69416e7.png +2023-07-12 01:03:35,836 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qtgtsad0-media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 01:03:37,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:38,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:03:40,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:42,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:43,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:44,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:03:44,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:44,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:03:44,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:46,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:48,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:50,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:53,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:53,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:03:53,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:03:53,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:03:53,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:03:54,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:03:54,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:56,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:58,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:58,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:00,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:00,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:02,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:03,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:04,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:04,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:04,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:04,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:04,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:04,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:08,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:08,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:04:08,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:10,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:12,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:14,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:14,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:14,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:14,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:14,765 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:15,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:15,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:16,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:16,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:18,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:19,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:20,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:24,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:25,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:26,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:26,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:26,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:26,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:26,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:26,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:28,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:30,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:30,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:30,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:32,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:34,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:36,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:36,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:36,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:36,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:36,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:36,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:38,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:38,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:04:40,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:41,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:42,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:46,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:46,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:46,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:47,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:47,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:47,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:47,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:47,300 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:48,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:48,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:50,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:52,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:53,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:54,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:57,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:57,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:57,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:57,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:58,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:58,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:58,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:00,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:01,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:01,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:02,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:03,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:04,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:07,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:05:07,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:05:07,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:05:07,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:05:08,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:05:08,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:08,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:05:08,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:10,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:12,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:05:12,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:05:12,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:05:12,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:05:12,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:05:12,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:13,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:14,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:16,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:16,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:16,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:18,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:23,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:28,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:31,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:31,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:33,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:38,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:05:39,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:44,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:46,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:46,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:49,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:54,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:59,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:01,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:01,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:05,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:06,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_96a516914a407fcfc504.png with policy now +2023-07-12 01:06:06,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_7a35da35c250d174b078.png with policy now +2023-07-12 01:06:06,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_8833dd1eeb120dd86be3.png with policy now +2023-07-12 01:06:06,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_cc24f9252c2efc365a1d.png with policy now +2023-07-12 01:06:06,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:06,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:06,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:06,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:06,711 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvxptxya-media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 01:06:06,760 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4i1d0nhb-media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 01:06:06,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:06,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 01:06:06,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 01:06:06,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 01:06:06,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 01:06:06,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:06:06,771 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ez37az7u-media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 01:06:06,803 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fa8i2bl8-media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 01:06:07,189 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m789jqjd-media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 01:06:07,203 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p4u4rz2y-media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 01:06:07,211 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lcjsx893-media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 01:06:07,247 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bvsjxzav-media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 01:06:08,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:06:11,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:12,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:14,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:16,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:16,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:16,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:16,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:16,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:20,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:22,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:22,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:24,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:26,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:26,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:26,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:26,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:26,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:26,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:28,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:28,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:30,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:31,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:31,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:33,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:34,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:35,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:35,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:35,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:35,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:36,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:36,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:38,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:06:38,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:38,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:40,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:42,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:44,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:46,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:46,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:46,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:46,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:46,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:46,846 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:46,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:46,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:48,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:50,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:50,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:54,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:55,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:56,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:56,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:56,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:56,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:56,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:56,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:58,872 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:00,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:00,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:02,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:02,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:02,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:05,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:05,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:05,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:05,463 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:06,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:06,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:06,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:08,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:07:08,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:10,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:11,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:14,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:16,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:16,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:16,353 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:16,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:16,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:16,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:17,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:17,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:17,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:20,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:22,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:22,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:24,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:26,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:26,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:26,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:26,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:26,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:26,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:27,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:30,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:32,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:32,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:32,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:33,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:34,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:37,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:37,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:37,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:37,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:38,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:07:38,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:38,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:38,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:40,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:42,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:42,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:42,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:42,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:42,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:42,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:43,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:44,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:47,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:47,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:49,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:54,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:59,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:02,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:02,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:04,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:08,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:08:09,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:14,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:17,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:17,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:20,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:25,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:30,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:32,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:32,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:35,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:36,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_9d432cf0ad1538825247.png with policy now +2023-07-12 01:08:36,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_c5ef4fbb8722ad5da91d.png with policy now +2023-07-12 01:08:36,937 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,937 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_4f96826bcb36d428d2d7.png with policy now +2023-07-12 01:08:36,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,938 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_c2fb4e9073164765d619.png with policy now +2023-07-12 01:08:36,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:08:36,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:08:36,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:08:36,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:08:37,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:08:37,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 01:08:37,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 01:08:37,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 01:08:37,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 01:08:37,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:08:37,226 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6haw5iwl-media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 01:08:37,235 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9f635joy-media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 01:08:37,255 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a90jd428-media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 01:08:37,287 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/usnqizk7-media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 01:08:37,618 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m0lz9qb8-media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 01:08:37,943 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cyjqvasr-media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 01:08:38,087 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1olrcr29-media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 01:08:38,279 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1p09kbfm-media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 01:08:38,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:08:41,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:41,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:43,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:45,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:46,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:46,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:08:47,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:48,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:48,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:49,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:52,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:53,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:55,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:55,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:08:55,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:08:55,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:08:55,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:08:57,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:08:57,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:57,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:59,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:01,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:03,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:03,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:03,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:05,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:05,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:05,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:05,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:05,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:07,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:07,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:08,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:08,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:09:09,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:13,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:14,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:15,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:16,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:16,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:16,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:16,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:17,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:17,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:18,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:18,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:19,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:19,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:21,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:25,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:25,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:26,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:26,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:26,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:26,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:27,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:27,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:29,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:30,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:31,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:33,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:33,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:35,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:35,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:36,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:36,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:36,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:36,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:37,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:37,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:38,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:09:39,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:40,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:43,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:45,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:46,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:46,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:46,898 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:46,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:46,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:47,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:47,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:48,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:48,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:51,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:51,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:53,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:55,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:56,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:56,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:56,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:56,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:57,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:57,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:01,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:02,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:03,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:04,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:04,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:05,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:07,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:10:07,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:07,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:10:07,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:10:07,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:10:08,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:10:09,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:10:09,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:11,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:12,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:10:12,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:12,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:10:12,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:10:12,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:10:13,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:10:13,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:15,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:18,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:19,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:19,542 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:23,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:28,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:33,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:34,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:34,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:38,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:10:38,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:43,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:48,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:49,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:49,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:53,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:58,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:03,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:04,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:04,761 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:07,001 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,001 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_3fe25e6178ac8d39607e.png with policy now +2023-07-12 01:11:07,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_630c030e959854fa6506.png with policy now +2023-07-12 01:11:07,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_5145710341bd620359f3.png with policy now +2023-07-12 01:11:07,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_18e1dc0d33fcdedb1fc5.png with policy now +2023-07-12 01:11:07,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:07,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:07,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:07,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:07,323 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fsx9p9w-media/images/validation_270_630c030e959854fa6506.png +2023-07-12 01:11:07,337 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kco4vrwe-media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 01:11:07,340 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x9rtpzoz-media/images/validation_270_5145710341bd620359f3.png +2023-07-12 01:11:07,397 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d6t71fj9-media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 01:11:07,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:07,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 01:11:07,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_630c030e959854fa6506.png +2023-07-12 01:11:07,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 01:11:07,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_5145710341bd620359f3.png +2023-07-12 01:11:07,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:11:07,799 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3wj9f05j-media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 01:11:07,804 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ibcr6is-media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 01:11:07,810 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tbf4s9d8-media/images/validation_270_630c030e959854fa6506.png +2023-07-12 01:11:08,089 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v00suy47-media/images/validation_270_5145710341bd620359f3.png +2023-07-12 01:11:08,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:11:09,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:11,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:13,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:15,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:15,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:16,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:17,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:19,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:19,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:20,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:21,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:23,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:25,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:26,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:26,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:26,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:26,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:26,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:27,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:27,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:31,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:31,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:33,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:34,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:34,761 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:35,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:36,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:37,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:37,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:37,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:37,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:38,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:11:39,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:39,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:41,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:42,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:43,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:47,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:47,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:47,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:47,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:47,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:47,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:49,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:49,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:49,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:49,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:51,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:53,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:55,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:57,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:58,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:58,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:58,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:58,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:59,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:59,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:59,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:01,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:03,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:04,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:04,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:04,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:05,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:07,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:07,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:07,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:07,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:07,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:08,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:12:10,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:10,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:12,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:14,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:16,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:16,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:17,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:17,047 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:17,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:17,049 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:18,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:18,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:19,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:19,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:21,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:22,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:24,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:27,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:28,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:28,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:28,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:28,054 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:28,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:30,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:30,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:32,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:32,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:34,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:35,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:35,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:37,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:38,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:38,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:38,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:12:38,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:38,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:38,655 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:40,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:40,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:42,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:43,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:43,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:43,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:43,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:43,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:44,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:44,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:46,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:48,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:50,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:50,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:54,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:59,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:04,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:05,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:05,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:08,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:13:09,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:14,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:19,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:20,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:20,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:25,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:30,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:35,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:35,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:36,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:37,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_4a948e463af013031d1f.png with policy now +2023-07-12 01:13:37,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_ef3aba43d9e8e71c9284.png with policy now +2023-07-12 01:13:37,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,340 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_c25f63eee5ef9870fbd4.png with policy now +2023-07-12 01:13:37,351 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_c142b41b461357b2de28.png with policy now +2023-07-12 01:13:37,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:13:37,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:13:37,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:13:37,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:13:37,656 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iop0n8hy-media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 01:13:37,664 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ot513gul-media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 01:13:37,664 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/77zszpwi-media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 01:13:37,676 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i2rp4v7x-media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 01:13:38,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:13:38,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 01:13:38,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 01:13:38,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 01:13:38,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 01:13:38,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:13:38,541 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpwy4och-media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 01:13:38,570 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/znar4tgh-media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 01:13:38,576 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/392asd5d-media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 01:13:38,579 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tyeabbhy-media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 01:13:38,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:13:42,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:42,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:44,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:46,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:47,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:47,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:13:50,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:51,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:51,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:52,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:52,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:54,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:56,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:57,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:13:57,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:13:57,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:13:57,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:13:58,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:58,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:13:58,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:00,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:03,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:04,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:06,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:06,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:06,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:06,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:06,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:06,858 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:06,860 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:08,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:08,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:08,637 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:14:09,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:10,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:14,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:15,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:16,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:16,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:16,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:16,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:16,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:18,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:18,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:20,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:22,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:22,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:22,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:24,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:26,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:26,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:28,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:28,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:28,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:28,144 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:28,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:30,315 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:31,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:32,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:34,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:36,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:37,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:37,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:37,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:37,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:37,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:38,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:38,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:38,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:38,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:14:40,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:42,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:42,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:44,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:47,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:48,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:48,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:48,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:48,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:48,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:52,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:52,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:52,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:52,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:54,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:58,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:58,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:59,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:59,245 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:59,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:59,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:15:00,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:15:00,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:02,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:06,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:08,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:08,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:15:08,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:08,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:08,804 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:09,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:15:09,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:15:09,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:15:09,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:15:10,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:15:10,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:13,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:14,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:15:14,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:15:14,165 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:15:14,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:15:14,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:15:14,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:16,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:19,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:23,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:23,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:25,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:30,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:35,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:38,640 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:15:38,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:38,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:40,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:45,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:50,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:54,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:54,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:56,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:01,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:06,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:08,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,314 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_ff574f4d8f5ce6c5ff1a.png with policy now +2023-07-12 01:16:08,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_b84e927620cd6b0cb2a8.png with policy now +2023-07-12 01:16:08,359 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_1c16b044a10bd74e136a.png with policy now +2023-07-12 01:16:08,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,361 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_1b1094343a18b1720c55.png with policy now +2023-07-12 01:16:08,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:08,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:08,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:08,391 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:08,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:08,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 01:16:08,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 01:16:08,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 01:16:08,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 01:16:08,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:16:08,612 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/esuty0jl-media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 01:16:08,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:16:08,675 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p8cw5ln0-media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 01:16:08,682 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xw1jzmzo-media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 01:16:08,700 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qwn58tv2-media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 01:16:09,009 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/75gemzuu-media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 01:16:09,028 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kvg81yi5-media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 01:16:09,042 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/soi9uhji-media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 01:16:09,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:09,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:09,336 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ku87wxs8-media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 01:16:11,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:12,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:14,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:16,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:18,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:18,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:20,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:22,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:22,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:24,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:24,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:24,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:26,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:27,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:27,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:27,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:27,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:28,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:28,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:28,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:32,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:33,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:34,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:37,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:37,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:37,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:37,074 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:38,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:16:38,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:38,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:39,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:39,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:39,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:40,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:44,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:44,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:46,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:47,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:47,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:47,352 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:47,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:48,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:48,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:50,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:52,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:54,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:54,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:54,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:55,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:56,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:57,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:57,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:57,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:57,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:58,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:58,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:00,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:00,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:04,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:05,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:06,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:07,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:07,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:07,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:07,725 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:08,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:17:08,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:08,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:09,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:09,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:17:11,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:12,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:14,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:16,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:16,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:18,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:18,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:18,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:18,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:18,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:20,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:21,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:22,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:24,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:24,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:17:24,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:26,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:27,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:27,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:27,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:27,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:27,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:28,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:28,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:32,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:33,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:34,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:36,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:37,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:37,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:37,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:37,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:38,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:17:38,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:38,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:38,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:40,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:40,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:17:42,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:42,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:42,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:42,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:42,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:42,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:44,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:44,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:49,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:54,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:55,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:55,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:00,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:05,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:08,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:18:10,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:10,275 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:10,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:15,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:20,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:25,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:25,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:25,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:30,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:35,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:37,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_a2de729812e74d6ab884.png with policy now +2023-07-12 01:18:37,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,416 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_872691b49d402d6a737f.png with policy now +2023-07-12 01:18:37,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_1dc75f8c8767180b4449.png with policy now +2023-07-12 01:18:37,437 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_1a544bb059e5c537308c.png with policy now +2023-07-12 01:18:37,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:18:37,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:18:37,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:18:37,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:18:37,725 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6k3q4jnq-media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 01:18:37,734 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9awc3om1-media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 01:18:37,765 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b6origw0-media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 01:18:37,779 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5qr01v9t-media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 01:18:38,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:18:38,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:18:38,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 01:18:38,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 01:18:38,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 01:18:38,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 01:18:38,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:18:39,295 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fzpkqyiw-media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 01:18:39,301 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/txg2q3vm-media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 01:18:39,321 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c8lk94mh-media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 01:18:39,325 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k8jo3dzv-media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 01:18:40,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:40,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:40,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:42,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:44,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:46,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:46,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:47,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:18:48,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:52,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:52,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:54,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:55,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:55,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:56,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:57,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:57,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:18:57,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:18:57,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:18:57,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:18:58,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:18:58,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:02,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:02,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:05,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:07,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:07,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:07,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:07,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:07,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:07,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:08,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:19:09,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:09,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:10,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:10,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:13,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:13,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:15,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:17,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:18,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:18,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:18,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:18,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:18,444 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:19,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:21,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:23,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:24,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:26,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:26,095 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:27,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:29,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:29,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:29,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:29,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:29,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:29,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:31,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:31,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:33,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:34,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:37,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:38,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:19:39,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:39,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:39,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:39,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:39,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:39,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:41,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:41,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:41,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:41,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:44,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:45,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:47,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:49,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:49,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:49,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:49,502 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:49,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:50,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:51,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:51,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:53,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:55,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:56,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:56,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:57,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:59,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:59,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:59,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:59,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:59,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:20:01,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:20:01,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:01,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:05,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:07,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:08,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:20:09,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:09,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:20:09,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:20:09,938 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:20:09,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:20:11,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:20:11,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:11,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:11,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:20:12,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:14,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:20:14,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:20:14,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:20:14,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:20:15,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:20:15,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:17,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:18,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:23,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:27,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:27,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:20:29,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:34,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:38,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:20:39,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:42,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:42,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:20:45,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:50,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:55,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:57,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:57,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:00,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:05,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:08,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:21:08,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:08,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_619cd7b6e4fcfa89810a.png with policy now +2023-07-12 01:21:09,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:09,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_5a10ea9a10a25e57135c.png with policy now +2023-07-12 01:21:09,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:09,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_43c929f6445607c5f8d2.png with policy now +2023-07-12 01:21:09,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:09,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_01b83aa2726efc837f38.png with policy now +2023-07-12 01:21:09,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:09,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:09,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:09,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:09,321 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/957kycom-media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 01:21:09,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:09,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 01:21:09,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 01:21:09,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 01:21:09,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 01:21:09,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:21:09,343 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t5jf5mtv-media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 01:21:09,355 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8hiirjsc-media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 01:21:09,362 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k8qg21yt-media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 01:21:09,713 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p62scj7h-media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 01:21:09,821 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nuwxwt50-media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 01:21:09,833 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cxn8401v-media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 01:21:10,067 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wxp8d9ml-media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 01:21:11,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:12,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:12,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:15,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:16,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:17,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:19,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:19,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:21,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:21,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:25,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:27,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:27,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:27,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:27,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:28,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:28,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:28,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:28,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:29,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:29,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:31,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:32,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:33,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:35,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:37,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:38,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:21:39,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:39,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:39,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:39,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:39,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:39,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:41,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:42,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:42,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:43,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:43,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:45,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:48,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:49,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:49,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:49,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:49,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:49,422 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:51,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:51,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:53,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:53,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:57,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:57,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:57,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:59,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:59,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:59,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:59,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:59,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:59,603 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:01,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:01,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:04,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:05,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:07,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:08,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:22:09,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:09,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:10,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:10,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:10,029 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:10,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:11,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:11,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:12,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:12,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:14,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:15,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:17,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:20,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:20,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:20,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:20,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:20,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:21,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:21,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:23,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:25,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:25,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:27,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:27,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:27,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:30,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:30,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:30,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:30,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:31,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:31,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:31,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:33,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:35,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:37,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:37,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:38,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:22:39,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:40,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:40,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:40,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:40,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:41,567 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:41,567 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:42,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:42,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:42,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:45,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:45,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:45,026 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:45,029 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:45,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:45,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:47,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:48,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:53,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:57,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:57,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:58,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:03,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:08,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:23:09,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:12,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:12,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:14,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:19,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:24,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:28,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:28,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:30,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:35,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:38,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:23:39,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_73ad6d9a88337e318469.png with policy now +2023-07-12 01:23:39,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,110 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_a373a58713d055f4d0ec.png with policy now +2023-07-12 01:23:39,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,120 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_71caeb46293b077a4cd4.png with policy now +2023-07-12 01:23:39,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_1627810819d85c5e49c5.png with policy now +2023-07-12 01:23:39,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:23:39,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:23:39,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:23:39,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:23:39,403 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bemzo8j1-media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 01:23:39,419 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/495ncr5b-media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 01:23:39,494 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rqvtcvo7-media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 01:23:39,520 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0oo3049m-media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 01:23:39,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:23:39,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 01:23:39,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 01:23:39,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 01:23:39,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 01:23:39,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:23:40,060 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4ssif8i-media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 01:23:40,086 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3mdjdm9f-media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 01:23:40,089 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6602igfz-media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 01:23:40,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:40,227 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5va3an5w-media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 01:23:43,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:43,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:43,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:45,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:45,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:48,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:23:49,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:51,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:51,751 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:53,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:55,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:57,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:58,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:58,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:59,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:23:59,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:23:59,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:23:59,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:23:59,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:23:59,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:01,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:02,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:03,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:05,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:08,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:08,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:24:09,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:09,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:09,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:09,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:09,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:09,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:11,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:13,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:13,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:14,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:14,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:17,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:18,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:19,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:20,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:20,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:20,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:20,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:21,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:21,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:23,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:25,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:27,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:28,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:29,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:29,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:29,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:30,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:30,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:30,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:30,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:31,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:31,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:34,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:35,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:37,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:38,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:24:39,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:39,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:40,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:40,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:40,159 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:40,161 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:41,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:41,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:43,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:44,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:44,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:44,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:47,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:49,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:49,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:49,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:49,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:49,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:49,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:50,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:51,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:53,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:56,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:57,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:59,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:59,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:59,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:00,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:25:00,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:25:00,309 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:25:00,311 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:25:01,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:01,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:25:01,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:05,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:06,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:07,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:08,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:25:09,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:25:09,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:25:09,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:25:09,773 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:25:09,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:25:09,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:11,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:11,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:13,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:14,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:25:14,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:25:14,260 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:25:14,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:25:14,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:25:14,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:25:15,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:25:15,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:17,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:17,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:22,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:27,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:29,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:25:29,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:25:32,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:37,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:38,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:25:43,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:44,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:25:44,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:25:48,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:53,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:58,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:00,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:00,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:04,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:08,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:26:08,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_4559efd7cd62cbc0d7bd.png with policy now +2023-07-12 01:26:08,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_fc7d6afb7198af162914.png with policy now +2023-07-12 01:26:08,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_7fbcfe2f1f50d8f10ad6.png with policy now +2023-07-12 01:26:08,767 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_103af67d972f2199c1fc.png with policy now +2023-07-12 01:26:08,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:08,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:08,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:08,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:09,095 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/db50nohj-media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 01:26:09,095 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wv89vtps-media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 01:26:09,095 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cruhfswn-media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 01:26:09,114 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dkduydz0-media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 01:26:09,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:10,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:10,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 01:26:10,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 01:26:10,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 01:26:10,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 01:26:10,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:26:10,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u3kgmjrh-media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 01:26:10,480 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f9omobly-media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 01:26:10,484 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oiyjfwr2-media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 01:26:10,491 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blsgy0yr-media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 01:26:14,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:15,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:15,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:15,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:16,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:18,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:18,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:20,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:22,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:25,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:26,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:28,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:28,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:28,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:28,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:28,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:30,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:30,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:30,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:30,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:32,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:34,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:36,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:36,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:38,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:38,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:38,402 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:38,402 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:38,404 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:38,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:26:40,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:40,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:42,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:44,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:45,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:45,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:46,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:48,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:48,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:48,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:48,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:48,552 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:50,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:50,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:53,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:54,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:56,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:58,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:59,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:59,511 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:59,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:59,514 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:00,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:00,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:00,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:00,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:02,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:04,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:04,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:08,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:08,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:27:09,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:09,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:09,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:09,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:09,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:10,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:10,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:12,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:14,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:14,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:15,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:15,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:18,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:19,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:20,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:20,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:20,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:20,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:20,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:22,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:22,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:24,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:25,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:28,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:30,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:30,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:30,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:30,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:30,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:30,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:30,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:30,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:30,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:32,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:36,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:36,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:38,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:38,669 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:27:40,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:40,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:40,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:40,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:40,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:40,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:41,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:42,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:45,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:45,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:45,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:45,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:45,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:45,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:46,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:46,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:46,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:48,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:51,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:56,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:00,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:00,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:02,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:07,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:08,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:28:12,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:15,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:15,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:18,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:23,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:28,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:30,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:30,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:33,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:38,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:38,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:28:39,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:39,973 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_f1d20714dd011423e767.png with policy now +2023-07-12 01:28:40,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:40,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_dc0f97f353b7e7ab6dca.png with policy now +2023-07-12 01:28:40,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:40,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_457d6bff4c2a7ae5f48a.png with policy now +2023-07-12 01:28:40,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:40,015 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_2a1bc7c87c68196530cf.png with policy now +2023-07-12 01:28:40,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:28:40,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:28:40,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:28:40,047 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:28:40,320 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o44o1d2l-media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 01:28:40,328 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eepiqkhw-media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 01:28:40,339 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4nilvx1p-media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 01:28:40,480 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fw8i0sc-media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 01:28:40,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:28:40,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 01:28:40,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 01:28:40,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 01:28:40,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 01:28:40,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:28:40,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w5id7nt3-media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 01:28:40,871 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dsmsynoh-media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 01:28:40,878 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o3zlux50-media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 01:28:40,885 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zul3r25-media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 01:28:43,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:45,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:45,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:46,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:48,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:48,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:50,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:28:50,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:52,564 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:54,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:56,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:59,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:59,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:28:59,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:28:59,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:28:59,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:00,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:00,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:01,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:01,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:02,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:04,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:04,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:06,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:08,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:29:09,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:10,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:10,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:10,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:10,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:10,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:10,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:12,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:14,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:14,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:16,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:16,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:16,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:19,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:19,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:19,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:19,887 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:19,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:20,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:20,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:22,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:24,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:25,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:28,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:30,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:30,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:30,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:30,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:30,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:30,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:30,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:32,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:32,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:32,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:35,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:36,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:38,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:29:38,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:40,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:41,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:41,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:41,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:41,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:41,253 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:42,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:44,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:46,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:46,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:48,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:48,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:48,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:50,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:51,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:51,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:51,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:51,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:52,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:52,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:54,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:56,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:58,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:58,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:00,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:00,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:30:00,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:30:00,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:30:00,887 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:30:02,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:30:02,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:03,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:03,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:03,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:06,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:08,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:30:08,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:08,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:10,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:10,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:30:10,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:30:10,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:30:10,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:30:12,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:30:13,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:14,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:16,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:30:16,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:30:16,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:30:16,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:30:16,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:30:16,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:18,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:18,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:18,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:19,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:20,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:24,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:29,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:33,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:33,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:35,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:38,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:30:40,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:45,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:48,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:48,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:51,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:56,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:01,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:03,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:03,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:07,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:08,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:31:11,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,366 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_22aac1153c97ec2693c5.png with policy now +2023-07-12 01:31:11,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_e283ca771222ebaf7b13.png with policy now +2023-07-12 01:31:11,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_bb4784ba59026990690d.png with policy now +2023-07-12 01:31:11,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,415 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_1c59f118bd1aa7b351ef.png with policy now +2023-07-12 01:31:11,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:11,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:11,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:11,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:11,679 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7zrw2eml-media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 01:31:11,717 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/66pbk1r6-media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 01:31:11,723 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5m9wumby-media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 01:31:11,740 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2d71trfl-media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 01:31:12,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:12,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:12,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 01:31:12,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 01:31:12,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 01:31:12,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 01:31:12,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:31:13,274 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grcofz36-media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 01:31:13,307 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wppdap8y-media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 01:31:13,315 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5a0b03r7-media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 01:31:13,334 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x96hd2ey-media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 01:31:16,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:18,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:18,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:18,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:18,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:20,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:21,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:23,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:24,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:26,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:28,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:28,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:31,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:31,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:31,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:31,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:31,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:33,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:33,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:33,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:33,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:35,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:38,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:31:38,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:39,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:40,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:40,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:40,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:40,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:41,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:41,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:43,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:44,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:47,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:48,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:48,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:49,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:50,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:52,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:52,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:52,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:52,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:53,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:53,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:55,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:55,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:57,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:01,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:01,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:02,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:02,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:02,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:02,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:03,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:03,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:04,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:04,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:05,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:07,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:08,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:32:09,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:11,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:12,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:12,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:12,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:12,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:12,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:13,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:13,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:15,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:17,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:19,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:20,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:20,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:21,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:22,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:23,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:23,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:23,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:23,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:23,729 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:25,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:27,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:27,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:29,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:31,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:32,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:33,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:33,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:33,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:33,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:33,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:35,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:35,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:35,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:37,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:38,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:38,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:32:39,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:41,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:43,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:43,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:43,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:43,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:43,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:43,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:45,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:47,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:47,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:47,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:47,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:47,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:48,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:49,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:49,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:50,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:50,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:51,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:54,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:59,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:04,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:05,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:05,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:08,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:33:09,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:14,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:19,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:20,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:20,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:24,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:29,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:34,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:35,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:35,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:38,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:33:40,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:41,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_26e2ebd54ecac5225fde.png with policy now +2023-07-12 01:33:41,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_d5ce8065e09a898bbb73.png with policy now +2023-07-12 01:33:41,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_1483311cc0a22899fd84.png with policy now +2023-07-12 01:33:41,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,971 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_09d003a6833146016be6.png with policy now +2023-07-12 01:33:42,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:33:42,001 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:33:42,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:33:42,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:33:42,240 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/snbslg5u-media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 01:33:42,287 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bwsi3t00-media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 01:33:42,299 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nuprcx0o-media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 01:33:42,303 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/af8uz5p9-media/images/validation_360_09d003a6833146016be6.png +2023-07-12 01:33:43,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:33:43,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 01:33:43,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 01:33:43,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_09d003a6833146016be6.png +2023-07-12 01:33:43,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 01:33:43,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:33:43,722 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wakw6gde-media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 01:33:43,746 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/14jl6msr-media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 01:33:43,760 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ib0rjqz3-media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 01:33:43,761 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icvuteh7-media/images/validation_360_09d003a6833146016be6.png +2023-07-12 01:33:45,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:47,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:49,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:50,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:50,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:50,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:51,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:33:51,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:55,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:56,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:57,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:59,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:01,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:01,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:01,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:01,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:01,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:01,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:01,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:05,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:05,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:05,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:07,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:07,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:08,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:34:09,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:11,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:11,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:11,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:11,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:11,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:11,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:12,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:15,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:17,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:17,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:19,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:21,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:21,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:21,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:21,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:21,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:21,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:21,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:22,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:23,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:25,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:27,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:28,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:29,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:31,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:31,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:31,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:31,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:33,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:33,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:33,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:35,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:36,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:36,993 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:37,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:38,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:34:39,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:41,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:42,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:42,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:42,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:42,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:43,552 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:43,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:44,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:45,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:47,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:50,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:51,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:52,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:52,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:52,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:52,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:52,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:52,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:53,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:53,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:55,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:56,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:59,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:01,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:01,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:02,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:35:02,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:35:02,746 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:35:02,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:35:03,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:35:03,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:05,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:07,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:07,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:07,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:08,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:35:09,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:11,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:12,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:13,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:35:13,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:35:13,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:35:13,090 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:35:13,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:35:13,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:15,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:17,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:35:17,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:17,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:35:17,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:35:17,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:35:17,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:35:19,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:22,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:22,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:23,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:28,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:33,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:37,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:37,482 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:38,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:38,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:35:43,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:48,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:52,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:52,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:53,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:58,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:03,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:07,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:07,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:08,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:36:09,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:11,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_66cc1fe983e4ffdf1c5b.png with policy now +2023-07-12 01:36:11,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_fa56e2608c00ceaca35e.png with policy now +2023-07-12 01:36:11,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_0b641787504f5e185899.png with policy now +2023-07-12 01:36:11,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_8ad08c11ab8ad22829d8.png with policy now +2023-07-12 01:36:11,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:11,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:11,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:11,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:11,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:11,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 01:36:11,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_0b641787504f5e185899.png +2023-07-12 01:36:11,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 01:36:11,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 01:36:11,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:36:11,813 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xs0fncrv-media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 01:36:11,848 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fu4qcgq-media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 01:36:11,858 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fvv0azz6-media/images/validation_370_0b641787504f5e185899.png +2023-07-12 01:36:11,860 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qk761rqn-media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 01:36:12,261 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f3uovqzy-media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 01:36:12,328 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zewmjw77-media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 01:36:12,611 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/77vnoo77-media/images/validation_370_0b641787504f5e185899.png +2023-07-12 01:36:12,659 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6br1axz-media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 01:36:15,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:15,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:17,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:19,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:20,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:21,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:22,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:22,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:23,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:25,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:25,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:27,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:29,846 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:30,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:30,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:30,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:30,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:31,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:31,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:31,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:33,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:37,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:37,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:37,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:37,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:38,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:36:39,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:40,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:40,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:40,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:40,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:41,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:41,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:42,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:43,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:45,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:48,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:49,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:50,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:50,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:50,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:50,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:51,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:51,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:53,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:53,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:53,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:55,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:57,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:59,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:59,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:00,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:00,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:00,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:00,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:01,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:01,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:04,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:05,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:07,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:08,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:08,352 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:08,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:37:09,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:10,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:11,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:11,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:11,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:11,208 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:11,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:11,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:15,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:16,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:17,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:19,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:21,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:21,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:21,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:21,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:21,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:22,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:22,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:23,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:23,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:26,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:26,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:28,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:30,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:31,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:31,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:31,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:31,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:32,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:32,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:34,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:36,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:38,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:38,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:37:38,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:38,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:38,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:40,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:41,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:41,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:41,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:41,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:42,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:43,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:44,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:46,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:46,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:46,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:46,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:46,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:48,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:48,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:49,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:53,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:53,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:54,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:59,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:04,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:08,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:38:08,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:08,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:09,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:14,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:19,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:23,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:23,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:25,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:30,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:35,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:38,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:38:39,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:39,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:40,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_8126eeaacd234695183b.png with policy now +2023-07-12 01:38:40,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_e3c3849a148163ace395.png with policy now +2023-07-12 01:38:40,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_150783a4c31c17364129.png with policy now +2023-07-12 01:38:40,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_cf0b270e00fd03447bc3.png with policy now +2023-07-12 01:38:40,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:38:40,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:38:40,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:38:40,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:38:41,057 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m6h2umbp-media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 01:38:41,079 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8xkpwe30-media/images/validation_380_150783a4c31c17364129.png +2023-07-12 01:38:41,084 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r2omgi7z-media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 01:38:41,090 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/455l7zqm-media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 01:38:41,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:42,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:38:42,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_150783a4c31c17364129.png +2023-07-12 01:38:42,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 01:38:42,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 01:38:42,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 01:38:42,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:38:42,602 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xaas1l9s-media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 01:38:42,616 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0g354fm5-media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 01:38:42,618 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ua05gi9h-media/images/validation_380_150783a4c31c17364129.png +2023-07-12 01:38:42,619 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1wskaieg-media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 01:38:44,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:47,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:48,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:50,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:50,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:38:52,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:52,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:54,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:54,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:54,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:56,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:58,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:58,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:59,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:38:59,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:38:59,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:38:59,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:00,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:02,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:03,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:04,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:06,279 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:08,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:08,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:39:09,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:09,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:09,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:09,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:09,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:09,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:09,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:10,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:10,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:12,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:14,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:16,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:18,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:19,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:19,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:19,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:19,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:20,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:20,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:20,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:24,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:24,208 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:24,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:25,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:26,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:28,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:30,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:30,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:30,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:30,201 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:30,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:31,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:32,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:34,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:36,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:37,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:38,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:38,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:39:39,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:39,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:40,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:40,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:40,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:40,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:40,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:42,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:42,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:44,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:46,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:47,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:50,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:50,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:50,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:50,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:50,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:52,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:52,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:52,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:54,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:54,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:54,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:58,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:58,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:00,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:01,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:40:01,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:40:01,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:40:01,545 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:40:02,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:40:02,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:03,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:06,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:08,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:08,699 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:40:08,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:09,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:09,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:10,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:12,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:40:12,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:40:12,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:40:12,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:40:12,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:40:14,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:14,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:16,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:17,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:40:17,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:40:17,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:40:17,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:40:18,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:40:18,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:19,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:24,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:24,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:25,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:30,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:35,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:38,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:40:39,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:39,562 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:40,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:45,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:50,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:54,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:54,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:55,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:00,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:05,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:08,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:41:09,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:09,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:10,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:11,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_4f13ece0060ef305e7c1.png with policy now +2023-07-12 01:41:11,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_cece94bd69f475704bfa.png with policy now +2023-07-12 01:41:11,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_b80214c7e7fafa67c5b0.png with policy now +2023-07-12 01:41:11,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_dea73b102aef933ebfb7.png with policy now +2023-07-12 01:41:11,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:11,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:11,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:11,627 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:11,845 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oppjbn4g-media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 01:41:11,850 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4giqmlqj-media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 01:41:11,865 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q12qum9w-media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 01:41:11,886 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m8fo4d41-media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 01:41:12,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:12,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 01:41:12,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 01:41:12,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 01:41:12,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 01:41:12,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:41:12,993 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1ehuax9-media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 01:41:13,003 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qnspzhnt-media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 01:41:13,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/woxvzgwk-media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 01:41:13,019 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ry33suft-media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 01:41:16,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:16,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:18,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:20,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:21,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:21,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:22,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:24,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:24,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:26,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:26,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:28,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:30,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:31,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:31,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:31,219 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:31,221 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:32,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:32,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:32,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:34,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:37,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:38,704 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:41:38,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:40,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:40,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:40,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:40,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:40,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:40,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:40,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:42,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:42,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:44,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:48,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:48,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:50,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:50,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:50,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:50,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:52,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:53,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:54,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:56,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:56,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:58,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:58,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:00,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:02,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:02,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:02,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:02,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:02,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:04,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:04,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:06,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:08,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:42:09,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:11,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:11,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:11,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:13,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:13,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:13,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:13,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:13,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:15,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:15,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:15,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:19,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:20,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:21,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:23,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:23,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:23,704 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:23,704 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:23,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:25,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:25,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:25,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:26,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:26,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:29,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:30,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:31,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:33,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:33,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:33,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:33,943 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:33,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:35,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:35,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:36,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:38,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:42:39,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:41,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:41,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:41,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:41,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:43,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:43,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:43,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:43,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:43,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:45,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:45,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:47,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:47,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:48,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:48,192 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:48,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:48,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:49,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:51,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:52,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:56,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:56,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:58,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:03,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:08,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:43:09,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:11,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:11,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:14,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:19,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:24,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:26,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:26,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:29,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:34,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:38,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:43:40,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:41,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:41,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:41,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,862 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_b8bc388d0a6aa1ae82dc.png with policy now +2023-07-12 01:43:41,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_301c182c9a0503b9c552.png with policy now +2023-07-12 01:43:41,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_d5487564dfd56ce35ceb.png with policy now +2023-07-12 01:43:41,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_5cbb16c19b007f3c49d3.png with policy now +2023-07-12 01:43:41,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:43:41,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:43:41,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:43:41,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:43:42,154 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0oohqkpu-media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 01:43:42,221 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kzc53a4z-media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 01:43:42,223 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sqkbvcsi-media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 01:43:42,223 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x6k645gi-media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 01:43:43,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:43:43,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 01:43:43,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 01:43:43,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 01:43:43,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 01:43:43,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:43:43,811 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mdlvoeb8-media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 01:43:43,821 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rjor2gp0-media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 01:43:43,824 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/00950s3e-media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 01:43:43,835 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3zyxzb6c-media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 01:43:46,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:47,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:49,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:51,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:51,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:52,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:43:53,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:56,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:56,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:57,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:57,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:59,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:00,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:00,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:00,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:00,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:01,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:01,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:03,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:03,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:05,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:07,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:08,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:44:08,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:09,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:10,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:10,400 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:10,400 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:10,402 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:11,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:12,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:12,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:44:13,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:14,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:15,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:17,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:19,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:20,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:20,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:20,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:20,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:20,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:21,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:23,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:25,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:26,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:27,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:27,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:27,696 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:44:30,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:30,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:30,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:30,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:31,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:31,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:31,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:33,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:35,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:37,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:37,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:38,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:44:40,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:40,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:40,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:40,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:41,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:41,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:42,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:42,696 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:44:43,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:43,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:45,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:47,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:49,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:50,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:50,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:50,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:50,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:51,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:53,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:54,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:55,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:57,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:57,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:57,971 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:00,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:01,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:45:01,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:45:01,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:45:01,261 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:45:01,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:45:01,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:03,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:05,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:06,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:07,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:08,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:45:11,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:11,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:11,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:45:11,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:45:11,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:45:11,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:45:13,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:45:13,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:14,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:14,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:16,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:17,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:45:17,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:45:17,029 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:45:17,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:45:17,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:45:17,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:19,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:22,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:27,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:29,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:29,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:33,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:38,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:38,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:45:43,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:44,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:44,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:49,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:54,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:59,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:59,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:59,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:04,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:08,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:46:09,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:11,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_36160e0e12a477c07ef9.png with policy now +2023-07-12 01:46:11,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_975e274aed0125eac405.png with policy now +2023-07-12 01:46:11,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_c88421f1a33201c27e7f.png with policy now +2023-07-12 01:46:11,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_3032692fd49e0f4ffa6a.png with policy now +2023-07-12 01:46:11,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:11,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:11,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:11,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:11,486 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpzv8jh1-media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 01:46:11,519 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n8bjgfj9-media/images/validation_410_975e274aed0125eac405.png +2023-07-12 01:46:11,533 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ei2lrzv7-media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 01:46:11,538 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f67wemps-media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 01:46:11,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:11,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 01:46:11,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 01:46:11,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 01:46:11,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_975e274aed0125eac405.png +2023-07-12 01:46:11,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:46:12,189 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sw3un76j-media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 01:46:12,191 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vb4grlcg-media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 01:46:12,193 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xwwx3gln-media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 01:46:12,198 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fpw5shcu-media/images/validation_410_975e274aed0125eac405.png +2023-07-12 01:46:14,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:14,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:15,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:15,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:17,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:20,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:21,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:21,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:23,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:25,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:26,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:27,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:29,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:29,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:29,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:31,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:31,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:31,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:31,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:31,898 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:32,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:33,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:35,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:37,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:37,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:38,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:46:39,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:40,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:40,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:40,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:40,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:41,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:43,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:43,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:44,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:44,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:45,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:47,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:48,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:49,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:50,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:50,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:50,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:50,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:51,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:53,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:53,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:55,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:57,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:59,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:59,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:59,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:59,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:00,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:00,406 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:00,406 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:00,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:01,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:01,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:04,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:05,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:07,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:08,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:47:10,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:10,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:10,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:10,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:10,581 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:10,583 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:12,044 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:14,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:15,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:47:15,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:47:15,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:16,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:18,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:20,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:21,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:21,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:21,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:21,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:22,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:22,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:24,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:26,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:26,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:30,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:30,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:47:30,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:47:31,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:31,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:31,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:31,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:31,956 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:32,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:32,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:34,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:36,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:38,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:38,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:47:40,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:42,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:42,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:42,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:42,124 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:42,126 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:42,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:44,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:44,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:46,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:47:46,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:47:46,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:46,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:46,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:46,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:47,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:48,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:48,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:50,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:52,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:57,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:01,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:01,270 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:03,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:08,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:08,720 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:48:13,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:16,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:16,270 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:19,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:24,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:29,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:31,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:31,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:34,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:38,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:48:39,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:41,324 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_2a287e605f741cc4ba84.png with policy now +2023-07-12 01:48:41,333 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,333 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_f17cfd3caf204c385f2c.png with policy now +2023-07-12 01:48:41,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_b5f6b2004b8fac7ef053.png with policy now +2023-07-12 01:48:41,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,353 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_e80fbc111f2744e80e0e.png with policy now +2023-07-12 01:48:41,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:48:41,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:48:41,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:48:41,395 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:48:41,635 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hh5uxqrf-media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 01:48:41,656 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02zi0f83-media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 01:48:41,662 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/78gdotye-media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 01:48:41,762 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/894vbkvv-media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 01:48:42,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:48:42,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 01:48:42,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 01:48:42,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 01:48:42,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 01:48:42,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:48:42,654 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1p5s7w36-media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 01:48:42,656 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3l897u41-media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 01:48:42,692 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ihfrbij7-media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 01:48:42,990 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7j9nyjin-media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 01:48:45,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:46,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:46,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:46,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:50,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:51,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:51,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:48:52,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:54,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:56,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:57,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:58,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:00,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:00,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:00,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:00,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:00,918 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:01,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:01,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:02,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:02,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:04,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:06,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:08,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:08,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:08,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:49:11,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:11,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:11,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:11,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:12,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:12,402 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:13,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:14,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:16,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:16,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:16,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:18,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:18,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:21,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:21,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:21,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:21,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:22,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:22,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:23,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:24,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:26,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:28,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:29,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:31,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:31,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:31,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:31,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:31,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:31,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:32,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:32,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:34,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:34,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:36,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:38,725 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:49:39,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:40,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:42,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:42,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:42,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:42,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:42,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:42,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:44,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:45,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:46,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:46,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:46,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:50,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:50,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:52,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:52,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:52,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:52,229 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:52,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:52,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:54,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:55,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:58,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:00,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:01,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:01,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:01,875 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:02,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:50:02,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:50:02,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:50:02,480 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:50:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:50:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:04,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:06,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:08,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:50:08,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:10,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:11,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:13,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:50:13,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:50:13,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:50:13,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:50:14,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:50:14,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:16,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:16,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:16,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:16,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:18,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:50:18,669 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:50:18,669 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:50:18,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:50:18,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:50:18,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:20,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:22,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:22,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:27,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:31,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:31,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:32,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:37,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:38,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:50:43,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:46,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:46,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:48,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:53,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:58,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:01,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:01,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:03,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:08,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:51:09,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:12,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,947 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_582c2e8324c4a214eee2.png with policy now +2023-07-12 01:51:12,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 01:51:12,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_31a8bb8062237a4e5e03.png with policy now +2023-07-12 01:51:12,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 01:51:12,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_cc52ea02c073b4adb746.png with policy now +2023-07-12 01:51:12,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 01:51:12,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_466a9d653c954a92a2d2.png with policy now +2023-07-12 01:51:12,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 01:51:12,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:51:13,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:13,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:13,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:13,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:13,296 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bb0yy0bx-media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 01:51:13,297 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pr4bkhz5-media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 01:51:13,306 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sm1o58l7-media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 01:51:13,328 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cw1slxpm-media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 01:51:13,798 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/emyq9upx-media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 01:51:13,804 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jdgoarta-media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 01:51:14,138 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nawlxknc-media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 01:51:14,167 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7oyckd7-media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 01:51:14,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:15,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:16,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:16,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:19,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:20,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:21,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:22,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:23,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:25,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:25,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:29,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:30,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:31,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:31,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:31,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:32,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:32,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:32,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:32,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:35,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:36,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:37,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:38,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:51:41,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:42,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:42,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:42,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:42,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:42,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:43,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:43,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:45,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:46,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:46,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:47,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:47,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:52,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:52,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:52,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:52,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:52,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:53,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:53,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:55,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:58,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:59,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:01,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:01,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:01,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:02,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:02,245 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:02,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:02,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:03,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:03,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:03,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:05,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:07,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:08,731 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:52:08,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:11,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:13,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:13,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:13,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:13,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:13,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:14,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:15,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:16,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:16,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:17,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:19,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:19,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:21,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:24,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:24,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:24,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:24,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:25,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:25,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:25,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:27,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:29,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:30,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:31,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:31,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:31,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:33,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:33,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:33,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:33,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:35,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:35,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:36,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:37,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:38,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:52:39,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:41,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:41,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:43,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:43,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:43,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:43,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:45,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:45,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:46,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:46,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:47,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:47,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:48,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:48,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:48,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:48,523 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:49,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:49,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:51,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:52,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:53,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:57,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:02,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:02,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:03,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:08,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:08,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:53:13,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:17,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:17,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:19,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:24,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:29,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:32,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:32,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:34,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:38,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:53:39,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:42,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,642 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_996fc8410c2def6b1ad3.png with policy now +2023-07-12 01:53:42,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_b7c3984877608e60dcc5.png with policy now +2023-07-12 01:53:42,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_37feadf7c6d79ab08bb8.png with policy now +2023-07-12 01:53:42,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_850de8645d4e0ec1b327.png with policy now +2023-07-12 01:53:42,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:53:42,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:53:42,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:53:42,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:53:42,969 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lia4zt2x-media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 01:53:42,987 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uy0gmsja-media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 01:53:42,997 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vzqf9me4-media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 01:53:43,003 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/za5jvqzn-media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 01:53:43,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:53:43,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 01:53:43,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 01:53:43,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 01:53:43,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 01:53:43,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:53:43,805 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qduugorx-media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 01:53:43,808 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zqq92uns-media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 01:53:43,812 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0osuufyo-media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 01:53:43,909 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xubkmlkf-media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 01:53:45,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:47,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:47,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:47,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:49,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:51,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:51,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:53,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:53:55,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:56,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:57,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:59,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:01,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:02,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:02,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:02,402 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:02,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:02,405 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:02,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:02,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:03,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:03,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:05,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:07,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:07,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:08,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:54:11,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:11,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:11,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:11,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:11,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:13,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:13,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:13,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:15,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:17,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:17,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:17,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:19,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:21,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:22,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:22,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:22,256 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:22,258 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:23,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:23,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:25,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:25,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:29,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:30,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:31,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:32,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:32,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:32,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:32,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:32,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:32,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:33,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:33,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:35,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:37,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:38,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:54:39,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:40,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:43,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:43,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:43,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:43,638 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:43,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:43,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:45,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:46,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:47,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:48,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:48,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:49,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:51,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:52,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:53,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:53,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:53,677 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:53,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:53,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:55,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:57,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:57,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:59,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:01,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:02,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:03,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:03,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:03,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:55:03,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:55:03,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:55:03,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:55:03,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:55:05,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:07,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:08,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:08,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:55:09,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:13,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:13,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:14,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:55:14,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:55:14,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:55:14,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:55:15,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:55:15,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:17,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:18,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:18,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:18,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:19,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:55:19,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:55:19,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:55:19,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:55:19,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:55:21,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:23,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:28,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:33,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:33,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:34,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:38,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:55:39,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:44,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:48,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:48,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:49,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:59,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:03,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:03,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:04,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:08,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:56:10,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:13,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_62e3070332b5e7b50721.png with policy now +2023-07-12 01:56:13,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_129489423a36c578b83b.png with policy now +2023-07-12 01:56:13,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_0be657c2999bb3f2f0e8.png with policy now +2023-07-12 01:56:13,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,765 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_75c8cb1b49d83b55316d.png with policy now +2023-07-12 01:56:13,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:13,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:13,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:13,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:13,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:13,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 01:56:13,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 01:56:13,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 01:56:13,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_129489423a36c578b83b.png +2023-07-12 01:56:13,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:56:14,050 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o27mgdgc-media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 01:56:14,066 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xxklbi7r-media/images/validation_450_129489423a36c578b83b.png +2023-07-12 01:56:14,077 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yfo3n2um-media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 01:56:14,096 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sa699x8z-media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 01:56:14,429 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlk1ynf7-media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 01:56:14,458 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rfnhwf9b-media/images/validation_450_129489423a36c578b83b.png +2023-07-12 01:56:14,750 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p5z5f5d0-media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 01:56:14,783 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0of647xh-media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 01:56:15,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:18,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:18,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:19,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:21,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:21,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:24,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:25,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:26,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:27,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:30,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:32,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:32,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:34,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:34,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:34,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:34,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:34,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:34,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:34,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:36,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:36,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:38,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:38,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:38,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:56:42,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:43,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:43,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:43,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:43,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:43,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:44,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:44,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:46,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:48,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:49,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:49,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:49,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:50,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:54,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:54,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:54,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:54,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:54,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:54,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:56,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:56,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:58,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:00,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:00,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:04,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:04,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:04,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:04,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:07,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:07,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:07,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:07,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:08,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:08,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:57:10,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:12,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:13,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:14,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:14,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:14,729 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:14,730 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:14,731 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:16,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:16,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:18,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:19,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:19,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:20,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:22,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:23,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:24,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:24,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:24,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:24,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:24,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:26,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:26,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:28,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:29,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:32,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:34,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:34,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:34,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:34,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:35,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:35,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:35,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:35,196 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:36,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:36,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:38,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:57:39,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:40,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:42,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:44,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:44,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:44,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:44,717 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:44,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:45,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:46,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:46,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:49,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:49,564 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:49,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:49,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:49,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:49,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:50,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:50,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:51,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:52,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:56,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:01,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:04,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:04,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:06,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:08,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:58:11,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:16,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:19,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:19,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:22,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:27,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:32,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:34,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:34,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:37,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:38,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:58:42,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:44,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_a6c4a260b23627355952.png with policy now +2023-07-12 01:58:44,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_69bab95618e258fac08d.png with policy now +2023-07-12 01:58:44,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_de7722337cbd652fed8a.png with policy now +2023-07-12 01:58:44,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_387823557d6d5cb784bb.png with policy now +2023-07-12 01:58:44,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:58:44,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:58:44,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:58:44,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:58:44,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:58:44,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 01:58:44,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 01:58:44,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 01:58:44,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 01:58:44,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:58:44,581 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5twgfza8-media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 01:58:44,627 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i12k99im-media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 01:58:44,636 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l5wuygyo-media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 01:58:44,637 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tg2a2sql-media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 01:58:44,959 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iyznowyr-media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 01:58:45,079 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ddhl4rfv-media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 01:58:45,108 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kx6yjy59-media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 01:58:45,310 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n05eb11p-media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 01:58:48,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:48,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:49,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:49,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:50,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:53,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:53,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:58:54,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:56,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:58,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:59,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:00,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:02,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:03,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:03,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:03,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:03,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:04,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:04,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:05,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:05,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:05,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:08,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:08,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:59:10,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:11,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:14,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:14,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:14,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:14,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:14,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:14,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:16,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:16,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:18,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:20,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:21,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:21,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:21,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:24,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:24,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:24,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:24,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:24,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:26,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:26,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:26,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:28,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:31,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:32,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:34,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:34,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:34,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:34,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:34,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:36,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:37,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:37,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:38,748 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:59:40,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:42,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:42,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:44,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:45,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:45,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:45,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:45,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:46,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:46,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:47,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:48,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:52,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:52,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:52,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:52,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:54,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:55,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:55,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:55,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:55,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:56,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:56,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:57,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:00,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:02,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:03,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:04,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:05,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:00:05,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:00:05,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:00:05,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:00:06,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:00:07,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:07,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:08,751 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:00:08,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:08,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:10,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:12,780 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:14,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:14,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:16,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:00:16,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:00:16,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:00:16,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:00:16,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:00:18,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:20,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:20,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:00:20,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:00:20,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:00:20,736 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:00:20,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:00:20,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:22,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:22,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:22,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:25,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:30,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:35,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:37,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:37,588 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:38,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:00:40,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:45,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:50,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:52,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:52,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:56,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:01,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:06,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:07,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:07,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:08,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:01:11,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:14,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_f98b8202065d26c1ef2f.png with policy now +2023-07-12 02:01:14,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,805 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_ae66beb9ba6213202415.png with policy now +2023-07-12 02:01:14,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_729f9b8d68182ccc9b8c.png with policy now +2023-07-12 02:01:14,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_f3357ea3af4ad6b0deea.png with policy now +2023-07-12 02:01:14,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:14,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:14,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:14,844 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:15,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 02:01:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 02:01:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 02:01:15,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 02:01:15,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:01:15,106 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i8igoxlk-media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 02:01:15,125 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/knq2502k-media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 02:01:15,131 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1azt4lwt-media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 02:01:15,141 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ttz8z5c0-media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 02:01:15,456 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kk0ftgyr-media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 02:01:15,497 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ai8le3yf-media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 02:01:15,508 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fhqm9oz-media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 02:01:15,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k4fj52rc-media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 02:01:16,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:19,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:22,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:22,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:22,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:23,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:25,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:25,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:27,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:28,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:33,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:34,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:35,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:35,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:35,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:35,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:35,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:37,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:37,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:37,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:37,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:38,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:01:39,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:39,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:41,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:44,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:44,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:44,449 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:44,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:45,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:45,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:45,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:47,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:49,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:50,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:52,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:52,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:53,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:54,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:54,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:54,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:54,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:55,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:55,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:56,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:57,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:59,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:01,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:02,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:04,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:04,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:04,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:04,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:05,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:05,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:07,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:07,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:07,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:07,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:08,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:02:09,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:12,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:13,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:15,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:16,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:16,033 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:16,034 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:16,035 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:17,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:18,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:19,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:21,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:23,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:23,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:23,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:23,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:25,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:25,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:25,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:25,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:25,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:27,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:27,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:28,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:31,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:33,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:33,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:35,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:35,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:35,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:35,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:35,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:37,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:37,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:38,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:38,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:38,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:02:38,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:41,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:43,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:43,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:45,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:45,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:45,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:45,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:45,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:45,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:47,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:49,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:50,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:50,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:50,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:50,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:51,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:51,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:53,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:53,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:53,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:54,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:59,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:04,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:08,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:08,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:08,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:03:09,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:14,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:19,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:23,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:23,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:25,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:30,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:35,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:38,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:38,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:38,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:03:40,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:44,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_57793f0e6246b307afce.png with policy now +2023-07-12 02:03:44,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_36f12af806c3f1d50a42.png with policy now +2023-07-12 02:03:44,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_f389fd00788c87466ea2.png with policy now +2023-07-12 02:03:44,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_11e4b59458340be3e02c.png with policy now +2023-07-12 02:03:44,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:03:44,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:03:44,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:03:44,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:03:45,043 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/llqixg1w-media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 02:03:45,094 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jgesh98g-media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 02:03:45,119 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/94zh5aq4-media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 02:03:45,121 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uy9rc82m-media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 02:03:45,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:03:45,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 02:03:45,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 02:03:45,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 02:03:45,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 02:03:45,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:03:45,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:45,899 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c6vj5cms-media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 02:03:45,930 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2ueixcv0-media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 02:03:45,998 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l93wf6cj-media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 02:03:46,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2c1g795j-media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 02:03:49,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:51,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:51,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:53,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:53,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:54,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:03:55,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:56,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:57,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:59,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:01,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:02,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:03,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:03,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:03,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:03,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:03,736 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:05,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:05,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:08,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:08,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:08,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:08,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:04:09,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:11,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:13,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:14,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:14,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:14,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:14,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:14,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:15,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:17,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:19,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:19,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:21,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:23,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:23,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:23,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:24,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:24,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:24,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:24,189 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:25,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:25,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:27,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:29,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:30,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:31,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:34,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:34,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:34,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:34,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:35,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:35,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:35,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:37,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:38,759 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:04:39,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:39,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:39,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:41,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:43,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:45,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:45,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:45,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:45,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:45,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:45,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:46,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:47,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:49,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:52,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:53,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:54,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:54,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:55,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:55,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:55,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:55,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:55,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:55,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:57,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:57,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:59,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:03,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:03,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:05,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:06,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:05:06,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:05:06,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:05:06,407 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:05:07,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:05:08,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:05:08,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:09,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:09,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:09,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:11,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:13,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:13,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:15,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:16,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:05:16,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:05:16,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:05:16,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:05:17,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:05:19,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:19,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:21,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:05:21,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:05:21,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:05:21,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:05:21,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:05:21,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:23,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:24,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:24,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:24,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:25,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:29,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:34,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:38,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:05:39,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:39,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:40,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:45,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:50,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:54,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:54,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:55,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:00,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:05,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:08,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:06:09,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:09,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:15,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:16,028 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_e1df59c538a185bbc3ab.png with policy now +2023-07-12 02:06:16,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 02:06:16,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 02:06:16,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 02:06:16,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 02:06:16,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:06:16,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_bb00c284bf371c7cee01.png with policy now +2023-07-12 02:06:16,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_68abe5bccc0897c1b607.png with policy now +2023-07-12 02:06:16,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,074 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_47a8d03f0cc699178621.png with policy now +2023-07-12 02:06:16,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:16,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:16,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:16,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:16,337 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6zbwot3j-media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 02:06:16,371 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qsdmi0k8-media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 02:06:16,406 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bpwdpz3f-media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 02:06:16,448 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlymdhok-media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 02:06:16,824 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5n5omzl1-media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 02:06:16,837 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qamjx8ft-media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 02:06:16,859 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h2rs5wfa-media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 02:06:17,162 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yt8tlche-media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 02:06:18,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:21,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:21,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:24,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:24,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:24,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:24,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:27,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:27,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:30,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:32,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:33,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:33,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:33,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:33,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:33,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:36,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:36,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:38,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:38,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:06:39,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:40,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:40,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:42,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:44,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:44,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:44,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:44,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:45,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:45,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:49,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:51,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:54,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:55,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:55,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:55,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:55,306 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:55,308 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:55,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:55,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:57,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:57,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:00,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:00,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:03,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:06,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:06,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:06,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:06,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:06,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:06,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:08,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:07:09,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:09,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:10,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:10,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:12,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:12,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:15,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:16,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:16,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:16,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:16,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:17,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:18,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:21,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:22,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:24,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:26,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:26,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:27,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:27,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:27,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:27,192 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:27,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:27,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:28,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:30,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:33,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:33,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:37,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:37,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:37,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:37,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:38,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:38,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:07:39,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:39,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:41,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:41,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:42,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:44,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:45,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:47,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:47,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:47,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:47,465 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:48,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:48,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:49,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:51,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:52,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:52,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:52,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:52,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:54,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:56,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:56,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:57,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:00,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:05,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:08,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:08:10,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:11,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:11,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:16,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:21,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:26,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:26,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:26,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:31,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:36,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:38,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:08:41,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:41,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:42,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:46,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_03a01949302184444647.png with policy now +2023-07-12 02:08:46,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_0d74931567eda76fb139.png with policy now +2023-07-12 02:08:46,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,592 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_0d16f7c02c5cc655df6a.png with policy now +2023-07-12 02:08:46,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,603 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_89f5a8910a8c98de8cb9.png with policy now +2023-07-12 02:08:46,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:08:46,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:08:46,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:08:46,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:08:46,901 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nbtodcw8-media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 02:08:46,914 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gf0s2pf5-media/images/validation_500_03a01949302184444647.png +2023-07-12 02:08:46,927 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ozcl5cuy-media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 02:08:47,054 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/257d5qzc-media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 02:08:48,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:08:48,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 02:08:48,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_03a01949302184444647.png +2023-07-12 02:08:48,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 02:08:48,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 02:08:48,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:08:48,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:48,777 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n8tidecb-media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 02:08:49,002 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1clln2jp-media/images/validation_500_03a01949302184444647.png +2023-07-12 02:08:49,117 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6xxlzo5l-media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 02:08:49,136 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d40k6eq1-media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 02:08:51,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:54,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:54,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:56,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:56,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:56,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:08:57,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:59,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:00,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:03,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:05,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:06,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:06,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:06,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:06,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:06,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:06,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:08,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:09:09,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:11,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:11,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:11,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:12,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:15,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:16,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:16,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:16,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:16,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:16,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:18,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:18,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:21,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:22,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:24,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:26,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:26,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:26,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:26,588 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:26,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:26,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:27,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:27,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:27,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:30,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:33,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:33,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:36,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:36,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:36,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:36,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:36,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:36,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:38,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:09:38,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:39,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:41,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:41,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:42,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:44,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:45,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:47,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:47,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:47,551 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:47,553 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:48,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:48,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:49,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:51,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:54,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:54,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:56,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:56,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:57,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:57,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:57,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:57,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:57,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:57,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:00,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:00,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:03,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:06,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:06,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:07,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:10:07,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:10:07,249 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:10:07,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:10:08,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:10:09,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:10:09,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:11,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:12,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:12,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:12,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:15,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:17,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:18,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:10:18,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:10:18,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:10:18,043 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:10:18,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:10:18,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:21,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:22,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:22,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:10:22,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:10:22,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:10:22,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:10:24,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:10:24,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:27,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:27,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:27,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:28,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:33,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:38,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:38,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:10:42,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:42,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:43,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:48,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:53,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:57,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:57,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:59,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:04,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:08,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:11:09,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:12,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:12,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:15,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:16,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_3234686b4d18835995c9.png with policy now +2023-07-12 02:11:16,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_da96661dc21e252cae64.png with policy now +2023-07-12 02:11:16,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_34fac9bf4212cdd6d437.png with policy now +2023-07-12 02:11:16,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_e3e834df071a8745bc59.png with policy now +2023-07-12 02:11:16,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:16,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:16,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:16,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:17,245 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/850a2qea-media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 02:11:17,260 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgcp8qrj-media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 02:11:17,270 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kipxzq8l-media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 02:11:17,271 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4vy65n4f-media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 02:11:18,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:18,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 02:11:18,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 02:11:18,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 02:11:18,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 02:11:18,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:11:19,169 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kvfyyp72-media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 02:11:19,190 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/79czmzhx-media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 02:11:19,191 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zyi4gg64-media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 02:11:19,197 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bts2vy1u-media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 02:11:20,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:21,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:26,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:27,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:27,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:27,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:27,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:30,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:31,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:33,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:36,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:37,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:37,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:37,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:37,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:37,539 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:38,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:11:39,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:39,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:42,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:42,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:42,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:42,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:45,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:46,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:46,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:46,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:46,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:47,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:48,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:51,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:52,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:54,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:56,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:56,849 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:56,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:56,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:57,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:57,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:57,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:57,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:58,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:00,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:03,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:04,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:07,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:07,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:07,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:07,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:08,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:12:09,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:09,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:12,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:12,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:12,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:15,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:15,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:17,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:17,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:17,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:17,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:19,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:19,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:20,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:22,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:26,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:27,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:27,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:27,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:27,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:27,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:27,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:28,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:28,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:31,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:31,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:34,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:36,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:37,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:37,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:37,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:37,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:37,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:38,780 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:12:40,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:40,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:42,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:42,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:42,406 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:43,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:46,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:47,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:47,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:47,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:47,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:47,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:49,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:49,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:52,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:52,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:52,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:52,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:52,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:52,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:55,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:55,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:57,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:57,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:58,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:58,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:03,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:08,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:08,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:13:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:12,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:13,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:18,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:23,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:27,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:27,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:29,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:34,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:38,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:13:39,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:42,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:42,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:45,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:46,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_61f1eebc64efc60d8ae6.png with policy now +2023-07-12 02:13:46,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_8d7aa871f805bb466b02.png with policy now +2023-07-12 02:13:46,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_95ea6a7d25f4eeb9af52.png with policy now +2023-07-12 02:13:46,935 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_c82ca30fe9549b880c53.png with policy now +2023-07-12 02:13:46,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:13:46,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:13:46,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:13:46,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:13:47,214 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s3u0o8c4-media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 02:13:47,264 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o1ajwky8-media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 02:13:47,264 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bfq67dui-media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 02:13:47,268 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5otvxrun-media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 02:13:49,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:13:49,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 02:13:49,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 02:13:49,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 02:13:49,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 02:13:49,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:13:49,555 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lmepa3aa-media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 02:13:49,563 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xlalgt2q-media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 02:13:49,565 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u74qu6uz-media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 02:13:49,574 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kd1pdrbf-media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 02:13:50,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:52,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:13:55,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:13:56,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:57,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:13:57,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:57,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:58,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:01,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:02,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:04,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:06,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:06,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:06,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:06,677 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:07,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:07,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:07,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:08,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:14:10,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:12,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:13,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:16,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:17,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:17,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:17,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:17,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:18,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:19,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:22,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:23,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:25,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:26,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:26,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:26,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:26,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:27,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:28,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:28,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:29,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:34,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:34,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:37,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:37,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:37,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:37,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:37,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:37,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:38,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:14:39,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:40,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:42,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:42,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:43,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:45,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:46,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:47,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:47,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:47,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:47,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:49,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:49,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:50,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:52,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:55,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:56,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:58,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:58,075 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:58,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:58,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:58,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:58,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:58,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:01,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:01,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:01,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:04,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:07,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:07,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:08,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:15:09,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:15:09,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:15:09,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:15:09,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:10,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:10,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:12,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:13,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:13,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:15:16,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:18,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:18,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:15:18,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:15:18,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:15:18,959 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:19,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:19,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:22,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:23,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:15:23,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:15:23,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:23,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:15:23,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:25,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:25,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:28,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:28,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:15:33,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:38,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:15:39,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:43,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:15:44,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:49,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:54,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:58,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:58,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:00,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:05,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:08,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:16:11,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:13,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:13,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:16,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:17,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,942 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_d9608828cb52fcf1ad77.png with policy now +2023-07-12 02:16:17,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_a5b58d688f92b4fc1442.png with policy now +2023-07-12 02:16:17,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_001908c5ae0df54582c9.png with policy now +2023-07-12 02:16:17,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,987 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_9ebd350e55790857624d.png with policy now +2023-07-12 02:16:18,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:18,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:18,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:18,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:18,310 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jdpwucqz-media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 02:16:18,326 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0o3gz6pf-media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 02:16:18,326 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kf59cwr6-media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 02:16:18,328 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/drjzb9d8-media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 02:16:19,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:19,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 02:16:19,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 02:16:19,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 02:16:19,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 02:16:19,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:16:19,916 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qjz8pflj-media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 02:16:19,942 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/laichacm-media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 02:16:19,944 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d2yjwtr2-media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 02:16:19,964 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3xkefalr-media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 02:16:22,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:22,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:25,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:28,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:28,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:28,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:28,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:31,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:33,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:37,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:38,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:38,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:38,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:38,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:38,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:38,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:16:40,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:40,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:43,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:43,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:43,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:48,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:48,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:48,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:48,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:49,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:49,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:49,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:52,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:54,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:55,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:58,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:58,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:58,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:58,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:58,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:58,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:58,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:58,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:59,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:01,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:04,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:05,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:07,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:07,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:07,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:07,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:07,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:08,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:17:10,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:10,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:10,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:13,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:13,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:13,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:17:15,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:16,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:17,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:17,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:17,843 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:17,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:19,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:19,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:21,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:22,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:25,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:27,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:28,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:28,422 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:28,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:28,425 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:28,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:28,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:28,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:17:31,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:32,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:34,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:37,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:37,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:38,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:38,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:38,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:38,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:38,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:17:40,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:40,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:43,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:43,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:43,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:17:48,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:48,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:48,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:48,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:48,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:49,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:49,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:52,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:53,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:53,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:53,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:53,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:54,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:55,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:55,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:58,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:58,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:00,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:05,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:08,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:18:10,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:14,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:14,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:16,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:21,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:26,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:29,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:29,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:31,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:36,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:38,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:18:41,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:44,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:44,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:47,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:48,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_978f41b570f2d5fd48e5.png with policy now +2023-07-12 02:18:48,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_68ef6d897da63a408d01.png with policy now +2023-07-12 02:18:48,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_37a3b637808816ae2a1f.png with policy now +2023-07-12 02:18:48,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_fc4fe59f596766ac60b6.png with policy now +2023-07-12 02:18:48,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:18:48,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:18:48,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:18:48,206 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:18:48,437 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zb796zei-media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 02:18:48,441 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4yajszcc-media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 02:18:48,469 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5jpa7rt3-media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 02:18:48,561 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vl1xh8gx-media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 02:18:50,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:18:50,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 02:18:50,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 02:18:50,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 02:18:50,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 02:18:50,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:18:50,331 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l7pr2mte-media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 02:18:50,332 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xz55n6o2-media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 02:18:50,410 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ecq2pqb5-media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 02:18:51,232 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/onqdushy-media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 02:18:52,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:53,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:18:56,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:18:58,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:58,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:18:59,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:18:59,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:59,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:02,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:03,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:05,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:08,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:08,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:08,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:08,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:08,625 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:08,626 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:08,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:19:11,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:11,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:14,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:15,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:19:15,319 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:18,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:18,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:18,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:18,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:19,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:20,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:20,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:23,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:25,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:26,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:28,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:28,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:28,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:28,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:29,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:30,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:30,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:19:30,319 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:32,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:35,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:36,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:38,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:38,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:38,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:38,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:38,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:38,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:19:41,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:41,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:41,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:44,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:45,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:19:45,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:47,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:48,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:48,720 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:48,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:48,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:50,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:50,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:52,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:53,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:56,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:58,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:59,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:59,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:59,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:59,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:59,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:01,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:01,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:02,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:02,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:03,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:05,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:08,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:08,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:08,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:20:09,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:20:09,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:20:09,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:20:09,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:11,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:11,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:13,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:14,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:16,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:19,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:20,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:20,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:20:20,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:20:20,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:20:20,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:23,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:23,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:25,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:25,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:20:25,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:20:25,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:20:25,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:26,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:26,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:29,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:31,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:31,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:31,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:36,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:38,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:20:42,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:46,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:46,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:47,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:52,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:57,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:01,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:01,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:02,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:07,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:08,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:21:13,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:16,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:16,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:18,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:19,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_90a56ad8ada0c9c8274d.png with policy now +2023-07-12 02:21:19,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_116ac864f0f13e57d81b.png with policy now +2023-07-12 02:21:19,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_55055c73d3e5ab4513f3.png with policy now +2023-07-12 02:21:19,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_402084574a959b5e4266.png with policy now +2023-07-12 02:21:19,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:19,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:19,462 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:19,464 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:19,706 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mvsrk80n-media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 02:21:19,731 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ho5irbyn-media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 02:21:19,731 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vennmvx3-media/images/validation_550_402084574a959b5e4266.png +2023-07-12 02:21:19,745 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/88w99fuv-media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 02:21:20,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:20,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 02:21:20,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 02:21:20,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_402084574a959b5e4266.png +2023-07-12 02:21:20,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 02:21:20,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:21:20,835 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blolfqm1-media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 02:21:20,846 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wxwcrkyy-media/images/validation_550_402084574a959b5e4266.png +2023-07-12 02:21:20,847 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02pixcbi-media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 02:21:20,856 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mx66wllq-media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 02:21:24,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:26,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:29,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:29,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:30,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:32,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:32,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:32,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:35,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:35,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:38,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:38,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:38,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:38,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:38,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:38,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:38,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:21:40,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:41,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:44,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:46,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:47,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:47,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:47,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:47,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:47,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:47,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:47,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:50,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:50,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:51,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:53,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:56,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:57,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:57,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:57,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:57,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:57,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:59,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:59,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:02,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:02,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:02,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:02,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:05,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:07,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:07,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:07,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:07,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:07,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:08,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:08,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:08,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:22:12,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:14,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:17,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:17,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:17,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:17,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:17,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:17,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:17,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:17,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:20,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:20,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:23,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:23,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:26,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:28,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:28,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:28,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:28,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:28,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:29,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:32,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:32,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:32,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:33,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:35,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:38,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:38,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:38,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:38,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:38,625 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:38,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:38,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:38,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:22:41,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:43,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:44,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:48,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:48,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:49,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:49,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:49,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:49,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:49,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:50,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:50,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:53,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:54,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:54,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:54,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:54,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:54,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:56,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:56,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:59,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:03,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:03,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:05,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:08,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:23:10,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:15,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:18,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:18,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:20,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:25,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:30,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:34,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:34,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:36,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:38,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:23:41,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:46,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:48,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_607b86b7a89c36fdadfe.png with policy now +2023-07-12 02:23:48,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_a72665b3636d99570f6e.png with policy now +2023-07-12 02:23:48,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_a54da6547e3af36c2510.png with policy now +2023-07-12 02:23:48,759 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_d160b3bcabae5e649acf.png with policy now +2023-07-12 02:23:48,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:23:48,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:23:48,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:23:48,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:23:49,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:49,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:49,039 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpm5tpm3-media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 02:23:49,065 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/91ttg7pm-media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 02:23:49,066 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/04ig92hf-media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 02:23:49,074 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h3bs5q0t-media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 02:23:50,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:23:50,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 02:23:50,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 02:23:50,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 02:23:50,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 02:23:50,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:23:51,152 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hj2ahtyq-media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 02:23:51,186 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6tsnlkzn-media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 02:23:51,188 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l0whpmyp-media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 02:23:51,196 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hi2eseub-media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 02:23:51,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:53,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:23:56,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:23:57,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:58,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:02,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:02,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:04,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:04,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:05,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:07,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:08,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:08,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:08,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:08,765 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:08,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:24:08,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:08,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:11,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:12,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:14,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:17,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:17,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:19,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:19,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:19,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:19,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:19,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:19,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:20,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:20,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:23,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:23,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:26,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:28,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:28,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:28,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:28,938 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:28,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:30,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:30,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:33,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:34,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:34,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:34,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:38,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:24:39,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:39,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:40,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:40,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:40,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:40,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:42,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:45,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:45,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:48,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:49,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:49,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:51,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:51,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:51,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:51,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:51,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:51,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:54,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:54,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:56,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:57,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:01,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:01,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:01,400 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:01,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:01,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:03,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:03,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:04,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:04,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:06,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:06,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:08,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:25:09,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:11,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:11,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:11,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:11,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:11,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:12,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:12,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:15,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:17,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:19,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:19,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:21,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:22,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:22,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:22,104 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:22,106 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:23,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:24,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:24,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:26,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:26,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:26,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:26,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:27,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:27,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:28,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:30,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:33,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:34,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:34,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:38,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:25:38,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:43,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:48,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:49,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:49,220 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:54,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:59,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:04,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:04,220 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:05,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:08,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:26:10,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:15,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:19,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:19,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:21,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:21,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_6d9eeb6998ef5daf6dc1.png with policy now +2023-07-12 02:26:21,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,433 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_eb681e116684d158f76a.png with policy now +2023-07-12 02:26:21,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_e92644ed701dc0b0faa2.png with policy now +2023-07-12 02:26:21,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_9c89c3127ee7dbbf4a2a.png with policy now +2023-07-12 02:26:21,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:21,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:26:21,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:26:21,463 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:26:21,714 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x9sjy590-media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 02:26:21,746 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ojyff4ky-media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 02:26:21,762 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c6xyct0y-media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 02:26:21,773 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o53k0ost-media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 02:26:24,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:26:24,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 02:26:24,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 02:26:24,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 02:26:24,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 02:26:24,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:26:24,678 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4pyuyb9x-media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 02:26:24,679 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpxrbg1o-media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 02:26:24,716 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6a9ex3qp-media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 02:26:24,737 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ljdp5rk1-media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 02:26:26,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:27,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:30,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:31,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:32,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:33,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:34,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:34,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:37,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:38,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:26:39,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:42,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:42,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:42,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:26:42,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:26:42,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:26:43,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:45,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:26:45,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:48,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:49,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:49,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:49,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:51,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:52,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:52,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:26:52,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:26:52,153 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:26:54,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:26:54,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:54,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:57,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:00,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:00,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:02,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:02,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:02,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:02,729 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:03,402 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:03,402 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:04,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:04,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:06,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:06,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:08,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:27:09,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:11,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:12,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:12,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:12,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:12,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:12,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:15,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:15,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:16,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:18,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:19,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:19,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:21,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:21,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:23,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:23,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:23,080 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:23,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:24,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:26,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:27,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:30,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:31,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:33,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:33,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:33,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:33,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:33,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:33,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:34,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:34,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:36,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:37,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:38,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:27:39,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:42,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:42,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:42,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:42,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:42,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:42,855 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:45,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:45,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:48,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:48,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:49,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:49,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:51,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:53,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:53,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:53,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:53,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:53,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:54,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:54,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:57,564 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:58,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:58,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:58,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:58,379 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:58,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:00,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:28:00,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:28:03,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:28:03,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:04,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:04,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:09,325 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:28:09,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:14,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:19,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:20,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:20,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:25,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:30,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:35,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:35,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:35,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:38,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:28:40,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:45,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:50,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:50,273 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:51,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:52,400 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_b26c685674fa1336fbab.png with policy now +2023-07-12 02:28:52,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_fd8f16b41072ce3bceb4.png with policy now +2023-07-12 02:28:52,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_b60bdd2ea4c17924c6d0.png with policy now +2023-07-12 02:28:52,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_d75cf57bdd0e29b1f7a8.png with policy now +2023-07-12 02:28:52,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:28:52,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:28:52,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:28:52,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:28:52,727 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zfekdgeu-media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 02:28:52,758 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l6sui735-media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 02:28:52,773 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p4ss4kp6-media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 02:28:52,780 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wrqx9t6g-media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 02:28:54,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:28:54,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 02:28:54,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 02:28:54,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 02:28:54,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 02:28:54,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:28:55,068 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ayatjk8t-media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 02:28:55,076 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/thvaky0z-media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 02:28:55,092 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hzcbjc7l-media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 02:28:55,102 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m6rwhrrd-media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 02:28:56,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:57,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:00,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:02,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:02,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:03,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:05,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:05,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:06,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:07,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:08,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:29:09,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:12,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:12,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:12,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:12,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:12,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:12,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:13,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:15,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:18,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:20,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:20,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:21,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:21,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:21,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:21,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:21,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:21,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:23,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:24,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:27,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:29,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:31,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:31,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:31,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:31,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:33,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:33,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:35,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:35,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:35,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:36,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:38,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:29:39,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:40,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:41,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:41,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:41,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:41,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:42,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:42,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:45,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:45,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:48,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:50,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:50,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:50,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:51,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:51,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:51,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:51,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:51,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:51,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:54,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:55,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:57,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:00,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:01,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:01,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:01,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:01,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:01,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:03,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:03,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:05,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:05,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:06,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:06,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:08,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:30:11,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:11,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:11,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:11,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:11,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:12,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:12,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:15,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:17,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:18,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:20,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:20,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:21,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:22,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:22,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:22,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:22,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:22,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:24,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:24,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:27,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:27,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:27,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:27,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:27,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:27,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:27,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:30,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:32,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:35,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:35,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:39,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:30:39,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:44,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:50,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:50,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:54,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:59,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:04,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:05,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:05,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:08,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:31:10,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:15,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:20,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:20,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:20,985 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:21,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_a98fc70c70656fdfe63b.png with policy now +2023-07-12 02:31:21,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_08863b59853c2669a10c.png with policy now +2023-07-12 02:31:21,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_e05be95e710336b098d2.png with policy now +2023-07-12 02:31:21,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_06bde4208de6769e990b.png with policy now +2023-07-12 02:31:21,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:21,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:31:21,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:31:21,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:31:21,828 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6b4kczj4-media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 02:31:21,872 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gtdikyrq-media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 02:31:21,886 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/anaa6mct-media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 02:31:21,890 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2jroihs6-media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 02:31:22,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:31:22,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 02:31:22,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 02:31:22,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 02:31:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 02:31:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:31:22,539 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yt24yq0f-media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 02:31:22,540 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/og5is7th-media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 02:31:22,542 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hg9czu5g-media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 02:31:22,550 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kqprf96v-media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 02:31:25,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:26,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:28,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:31,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:31,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:31,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:34,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:36,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:36,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:37,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:37,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:38,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:31:40,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:41,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:41,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:31:41,447 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:31:41,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:31:42,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:43,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:31:43,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:46,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:47,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:49,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:51,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:51,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:51,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:51,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:31:51,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:31:51,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:31:52,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:31:53,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:55,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:58,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:58,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:01,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:01,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:01,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:01,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:01,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:04,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:04,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:04,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:06,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:06,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:07,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:08,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:32:09,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:10,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:12,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:12,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:12,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:12,090 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:13,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:13,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:15,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:19,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:20,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:21,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:21,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:22,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:22,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:22,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:22,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:22,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:25,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:25,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:26,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:28,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:31,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:31,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:32,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:32,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:32,717 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:32,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:34,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:34,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:36,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:37,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:37,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:37,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:38,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:32:40,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:41,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:43,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:43,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:43,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:43,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:43,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:43,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:47,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:49,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:52,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:52,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:52,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:52,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:52,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:52,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:52,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:52,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:55,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:55,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:57,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:57,324 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:57,324 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:57,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:57,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:58,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:58,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:33:01,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:33:02,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:07,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:07,312 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:08,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:08,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:33:13,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:18,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:22,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:22,312 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:24,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:29,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:34,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:37,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:37,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:38,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:33:39,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:44,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:49,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:51,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_e5844c86288b2606b645.png with policy now +2023-07-12 02:33:51,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_07de5acd372977760b4c.png with policy now +2023-07-12 02:33:51,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_bbb685abfd63c223246e.png with policy now +2023-07-12 02:33:51,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_6b5551cfcbdcbf8c81e5.png with policy now +2023-07-12 02:33:51,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:33:51,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:33:51,742 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:33:51,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:33:52,067 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ouec5xcw-media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 02:33:52,108 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sqemhzyd-media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 02:33:52,109 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fjld9v7q-media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 02:33:52,114 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kw8y1nmt-media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 02:33:52,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:52,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:52,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:33:52,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 02:33:52,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 02:33:52,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 02:33:52,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 02:33:52,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:33:52,938 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ugmg0lv-media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 02:33:52,974 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lajohbtq-media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 02:33:53,025 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jj6cgpk3-media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 02:33:53,025 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6lz8nz57-media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 02:33:55,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:33:55,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:58,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:00,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:01,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:01,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:04,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:06,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:07,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:08,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:08,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:08,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:34:11,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:11,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:11,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:11,560 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:11,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:13,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:13,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:16,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:19,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:21,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:21,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:21,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:21,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:22,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:22,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:23,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:23,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:25,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:28,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:28,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:31,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:31,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:31,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:31,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:31,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:31,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:33,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:34,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:37,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:38,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:34:38,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:39,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:39,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:42,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:42,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:42,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:42,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:43,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:43,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:44,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:46,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:49,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:49,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:52,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:52,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:52,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:52,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:52,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:52,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:54,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:54,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:54,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:55,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:58,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:59,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:01,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:02,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:02,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:02,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:02,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:04,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:04,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:05,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:07,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:08,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:35:09,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:09,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:10,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:10,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:12,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:12,406 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:12,406 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:12,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:13,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:13,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:15,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:16,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:19,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:21,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:22,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:22,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:22,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:22,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:22,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:22,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:24,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:24,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:25,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:26,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:27,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:27,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:27,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:27,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:28,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:31,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:31,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:36,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:38,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:35:39,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:39,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:42,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:47,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:52,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:54,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:54,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:58,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:03,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:08,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:08,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:36:09,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:09,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:14,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:19,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:21,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_1d981ecc323c30e20b81.png with policy now +2023-07-12 02:36:21,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_b9cc12355deddaad9779.png with policy now +2023-07-12 02:36:21,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_134e6d2a749fcacd43b7.png with policy now +2023-07-12 02:36:21,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,694 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_92f0f0898c20a134b3c3.png with policy now +2023-07-12 02:36:21,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:21,725 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:36:21,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:36:21,727 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:36:21,939 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2mwrbwlo-media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 02:36:21,996 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wf5lgkyb-media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 02:36:22,017 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83o8qkul-media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 02:36:22,018 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3z8fh6at-media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 02:36:23,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:36:23,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 02:36:23,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 02:36:23,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 02:36:23,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 02:36:23,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:36:23,388 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u72w6ne9-media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 02:36:23,400 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l92cm2h0-media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 02:36:23,405 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3t1d354-media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 02:36:23,422 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b30f4ze6-media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 02:36:24,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:24,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:25,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:26,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:29,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:31,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:31,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:32,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:35,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:37,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:38,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:38,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:36:39,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:39,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:41,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:41,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:36:41,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:36:41,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:36:41,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:36:41,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:43,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:44,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:47,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:48,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:50,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:50,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:50,915 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:36:50,915 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:36:50,917 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:36:53,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:36:53,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:54,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:55,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:55,143 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:56,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:59,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:59,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:01,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:01,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:01,067 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:01,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:02,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:02,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:04,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:08,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:37:10,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:10,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:10,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:11,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:11,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:11,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:11,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:11,312 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:14,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:14,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:15,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:17,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:20,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:21,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:21,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:21,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:21,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:21,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:23,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:26,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:26,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:26,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:26,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:29,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:32,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:32,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:32,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:32,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:32,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:32,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:35,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:35,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:37,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:38,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:38,836 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:37:41,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:41,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:41,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:42,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:42,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:42,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:42,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:42,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:44,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:44,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:47,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:50,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:53,335 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:53,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:54,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:54,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:54,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:54,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:56,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:56,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:56,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:56,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:58,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:58,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:58,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:58,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:58,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:38:02,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:38:04,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:08,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:38:09,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:11,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:11,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:15,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:20,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:25,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:26,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:26,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:31,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:36,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:38,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:38:41,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:41,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:42,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:47,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:52,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:53,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_77093872b2a62f3184c0.png with policy now +2023-07-12 02:38:53,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_f72e5adad3a7e2192d4a.png with policy now +2023-07-12 02:38:53,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_ed7326150a0c47d8bd9c.png with policy now +2023-07-12 02:38:53,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_ad1a36db317705c6c410.png with policy now +2023-07-12 02:38:53,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:38:53,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:38:53,248 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:38:53,250 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:38:53,483 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o2s5o74y-media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 02:38:53,510 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/et9ww5li-media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 02:38:53,521 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n7rki5jp-media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 02:38:53,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:38:53,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 02:38:53,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 02:38:53,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 02:38:53,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 02:38:53,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:38:53,545 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z1yfbp4j-media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 02:38:53,875 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qgakst8k-media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 02:38:53,888 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9adx56pr-media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 02:38:53,891 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8zwxp8cj-media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 02:38:53,917 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wu8qdtdl-media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 02:38:56,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:56,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:58,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:59,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:02,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:03,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:04,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:05,582 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:08,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:08,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:39:09,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:11,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:11,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:11,598 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:13,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:13,278 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:13,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:13,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:14,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:14,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:15,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:17,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:20,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:21,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:22,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:22,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:22,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:22,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:23,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:23,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:26,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:26,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:26,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:26,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:32,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:32,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:32,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:32,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:32,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:32,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:35,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:35,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:37,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:38,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:38,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:39:41,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:42,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:42,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:42,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:42,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:42,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:42,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:42,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:44,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:44,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:47,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:48,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:50,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:53,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:53,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:53,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:53,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:53,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:53,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:53,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:56,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:57,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:57,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:59,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:59,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:02,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:03,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:03,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:03,877 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:03,879 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:04,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:05,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:08,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:08,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:40:10,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:11,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:12,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:12,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:14,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:14,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:14,094 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:14,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:14,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:14,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:16,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:17,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:20,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:21,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:23,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:23,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:23,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:23,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:23,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:23,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:26,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:26,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:27,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:27,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:28,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:28,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:28,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:28,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:29,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:29,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:32,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:32,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:37,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:38,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:40:42,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:42,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:43,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:48,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:53,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:57,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:57,229 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:58,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:03,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:08,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:08,844 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:41:12,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:12,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:41:13,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:18,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:22,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,885 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_4f603f3934c8321cc8fb.png with policy now +2023-07-12 02:41:22,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_79440b6bdd999b2e130a.png with policy now +2023-07-12 02:41:22,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_da9c8b5a7202534ddcc5.png with policy now +2023-07-12 02:41:22,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_bdab7dddcf554aad01f9.png with policy now +2023-07-12 02:41:22,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:22,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:41:22,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:41:22,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:41:23,198 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ayf2sld-media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 02:41:23,232 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ow023sp8-media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 02:41:23,246 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eeu1g7m5-media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 02:41:23,251 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ol8k73wz-media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 02:41:23,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:24,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:41:24,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 02:41:24,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 02:41:24,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 02:41:24,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 02:41:24,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:41:24,355 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/73z87ue7-media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 02:41:24,358 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyii1t3u-media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 02:41:24,387 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jldt6cx3-media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 02:41:24,407 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/21kfly09-media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 02:41:27,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:27,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:27,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:41:29,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:30,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:32,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:33,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:34,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:36,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:38,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:41:39,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:40,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:41,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:41,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:41:41,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:41:41,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:41:42,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:41:42,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:42,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:42,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:41:45,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:45,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:48,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:51,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:51,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:51,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:51,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:41:51,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:41:51,916 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:41:54,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:41:54,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:56,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:57,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:57,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:57,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:00,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:01,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:01,383 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:01,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:01,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:02,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:03,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:03,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:06,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:08,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:08,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:42:09,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:12,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:12,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:12,333 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:12,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:12,335 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:12,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:12,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:13,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:15,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:15,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:18,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:18,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:23,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:23,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:23,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:23,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:24,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:24,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:24,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:28,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:28,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:29,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:30,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:33,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:33,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:33,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:33,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:33,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:34,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:36,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:36,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:38,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:42:39,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:39,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:42,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:43,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:43,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:44,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:44,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:44,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:44,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:45,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:45,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:48,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:51,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:51,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:54,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:54,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:54,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:54,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:54,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:54,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:56,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:57,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:58,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:58,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:59,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:59,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:59,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:59,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:43:00,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:43:00,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:43:01,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:06,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:08,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:43:12,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:13,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:13,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:43:18,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:23,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:28,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:28,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:28,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:43:33,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:38,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:43:39,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:43,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:43,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:43:44,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:49,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:53,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,756 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_49779e88089791939e26.png with policy now +2023-07-12 02:43:53,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_b64ccfa16d3aa44a2151.png with policy now +2023-07-12 02:43:53,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_56616402a26a74e9ca6a.png with policy now +2023-07-12 02:43:53,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_c1d1fb2c049800d964ec.png with policy now +2023-07-12 02:43:53,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:43:53,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:43:53,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:43:53,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:43:54,089 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7rt9zrt3-media/images/validation_640_49779e88089791939e26.png +2023-07-12 02:43:54,127 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lx1270oc-media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 02:43:54,127 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7eyl13ks-media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 02:43:54,132 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p66i4dn5-media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 02:43:54,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:43:54,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 02:43:54,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_49779e88089791939e26.png +2023-07-12 02:43:54,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 02:43:54,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 02:43:54,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:43:54,893 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mi2ekolj-media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 02:43:54,904 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/63ez2e38-media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 02:43:54,910 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/do3r4q1q-media/images/validation_640_49779e88089791939e26.png +2023-07-12 02:43:54,921 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uu4cw65r-media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 02:43:55,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:58,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:58,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:00,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:01,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:02,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:03,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:06,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:07,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:08,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:44:09,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:11,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:11,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:11,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:11,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:12,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:12,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:12,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:13,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:13,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:17,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:18,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:21,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:22,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:22,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:22,063 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:22,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:23,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:24,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:24,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:27,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:28,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:30,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:33,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:33,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:33,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:33,075 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:33,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:33,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:34,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:36,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:38,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:44:39,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:42,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:43,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:43,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:43,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:43,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:43,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:43,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:44,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:45,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:45,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:48,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:50,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:51,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:53,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:53,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:53,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:53,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:54,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:54,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:55,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:59,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:59,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:00,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:01,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:03,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:03,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:03,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:03,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:03,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:06,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:06,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:06,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:08,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:45:09,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:11,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:12,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:14,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:14,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:14,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:14,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:14,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:14,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:15,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:15,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:16,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:18,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:21,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:22,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:24,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:24,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:24,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:24,200 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:24,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:24,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:27,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:28,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:29,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:29,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:29,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:29,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:29,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:29,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:30,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:30,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:33,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:38,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:45:39,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:44,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:44,303 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:45,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:50,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:55,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:59,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:59,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:01,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:06,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:08,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:46:11,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:14,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:46:14,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:17,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:22,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:23,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_7416b90b79fecff5e4ca.png with policy now +2023-07-12 02:46:23,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_606f3fe637934aa6a15b.png with policy now +2023-07-12 02:46:23,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_2b23cf43dc0d03af66c6.png with policy now +2023-07-12 02:46:23,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_774f3994c62fdf69ce65.png with policy now +2023-07-12 02:46:23,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:23,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:46:23,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:46:23,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:46:23,801 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bj28yw0b-media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 02:46:23,806 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h4gop8cg-media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 02:46:23,808 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gjor9qhd-media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 02:46:23,838 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/88xjqo48-media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 02:46:24,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:46:24,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 02:46:24,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 02:46:24,940 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 02:46:24,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 02:46:24,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:46:25,263 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/70mh8c9e-media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 02:46:25,277 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhe5luxj-media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 02:46:25,286 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/013kphde-media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 02:46:25,309 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dl90gqyz-media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 02:46:27,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:29,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:46:29,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:30,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:33,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:33,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:34,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:36,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:38,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:38,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:46:39,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:42,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:43,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:43,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:46:43,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:46:43,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:46:44,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:45,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:46:45,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:45,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:46:45,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:48,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:49,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:52,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:53,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:53,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:46:53,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:46:53,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:46:55,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:46:55,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:55,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:58,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:00,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:00,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:00,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:01,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:03,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:03,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:03,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:03,131 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:04,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:04,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:05,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:07,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:08,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:47:10,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:13,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:13,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:13,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:13,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:13,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:15,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:15,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:15,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:16,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:16,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:19,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:21,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:22,076 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:24,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:24,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:24,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:24,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:25,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:25,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:26,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:28,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:30,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:30,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:31,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:31,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:34,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:34,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:34,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:34,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:34,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:37,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:37,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:37,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:38,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:47:40,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:43,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:43,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:44,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:44,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:44,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:44,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:46,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:46,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:46,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:48,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:49,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:52,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:53,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:54,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:54,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:54,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:54,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:55,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:55,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:58,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:59,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:59,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:59,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:59,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:59,594 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:48:01,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:48:01,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:48:01,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:01,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:04,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:08,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:48:09,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:14,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:16,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:16,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:20,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:25,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:30,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:31,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:31,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:35,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:38,861 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:48:41,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:46,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:46,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:47,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:52,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:54,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_aca4a003a8d6b3b9f87b.png with policy now +2023-07-12 02:48:54,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,165 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_4155d6ba117c33d5628b.png with policy now +2023-07-12 02:48:54,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_65494724a21e7695e782.png with policy now +2023-07-12 02:48:54,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,167 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_3806edfc6615feb4c502.png with policy now +2023-07-12 02:48:54,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:48:54,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:48:54,193 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:48:54,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:48:54,450 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jp0k4gti-media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 02:48:54,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fhru8fuy-media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 02:48:54,502 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6o1k0gzt-media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 02:48:54,530 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/si6981io-media/images/validation_660_65494724a21e7695e782.png +2023-07-12 02:48:55,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:48:55,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 02:48:55,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 02:48:55,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 02:48:55,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_65494724a21e7695e782.png +2023-07-12 02:48:55,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:48:55,744 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/go8azig8-media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 02:48:55,756 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ejf51owy-media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 02:48:55,761 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2d2ncd3n-media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 02:48:55,766 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eniwcuwi-media/images/validation_660_65494724a21e7695e782.png +2023-07-12 02:48:58,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:01,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:01,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:01,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:03,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:04,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:04,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:07,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:08,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:49:08,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:13,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:13,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:13,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:13,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:13,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:13,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:14,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:16,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:17,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:17,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:19,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:19,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:23,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:23,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:23,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:23,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:25,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:25,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:25,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:28,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:30,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:31,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:32,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:32,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:34,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:34,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:34,195 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:34,197 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:34,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:34,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:36,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:37,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:38,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:49:40,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:41,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:43,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:44,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:44,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:44,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:44,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:46,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:47,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:47,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:47,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:49,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:52,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:52,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:54,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:54,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:54,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:54,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:55,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:55,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:58,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:58,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:01,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:02,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:02,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:04,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:04,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:04,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:04,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:04,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:04,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:07,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:07,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:08,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:50:09,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:10,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:13,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:14,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:14,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:14,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:14,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:14,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:17,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:17,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:19,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:19,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:22,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:24,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:24,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:24,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:24,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:25,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:25,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:25,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:28,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:29,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:29,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:29,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:29,344 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:30,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:31,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:31,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:32,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:32,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:36,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:38,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:50:41,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:46,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:47,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:47,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:52,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:57,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:02,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:02,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:03,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:08,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:08,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:51:13,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:17,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:17,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:19,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:23,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_00b528cb36f02641e0c4.png with policy now +2023-07-12 02:51:23,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_4b3b71fb783337ee170d.png with policy now +2023-07-12 02:51:23,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_a54a1c21b8282d62f8f9.png with policy now +2023-07-12 02:51:23,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_38a8a2cd9ee27689f5ea.png with policy now +2023-07-12 02:51:23,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:23,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:51:23,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:51:23,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:51:23,880 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2w5ejt9f-media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 02:51:23,943 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vazcpxha-media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 02:51:23,957 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7arzg2g6-media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 02:51:23,971 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ur9xor31-media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 02:51:24,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:25,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:51:25,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 02:51:25,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 02:51:25,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 02:51:25,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 02:51:25,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:51:26,298 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jujya9hd-media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 02:51:26,299 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rd1z54qt-media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 02:51:26,305 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sk0kogsn-media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 02:51:26,308 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyfxv51o-media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 02:51:28,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:30,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:31,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:32,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:32,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:33,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:34,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:35,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:37,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:38,868 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:51:40,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:40,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:43,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:43,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:51:43,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:51:43,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:51:43,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:51:43,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:46,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:46,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:47,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:47,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:49,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:52,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:53,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:53,243 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:51:53,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:51:53,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:51:55,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:51:55,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:57,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:58,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:01,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:02,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:02,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:02,851 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:03,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:03,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:03,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:03,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:04,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:04,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:07,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:08,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:08,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:52:10,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:13,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:13,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:13,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:13,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:13,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:14,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:14,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:17,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:18,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:18,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:18,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:20,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:23,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:23,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:23,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:23,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:23,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:26,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:26,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:29,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:29,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:33,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:33,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:33,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:33,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:33,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:33,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:34,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:35,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:35,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:38,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:38,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:52:39,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:41,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:43,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:43,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:43,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:43,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:44,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:44,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:45,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:47,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:48,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:48,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:51,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:53,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:53,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:53,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:53,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:53,981 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:56,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:56,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:56,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:59,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:59,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:59,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:59,026 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:59,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:59,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:53:02,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:53:02,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:03,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:03,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:07,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:08,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:53:12,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:17,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:18,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:18,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:23,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:28,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:33,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:33,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:34,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:38,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:53:39,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:44,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:48,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:48,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:50,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:53,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_7d58abcefc10987ef6e0.png with policy now +2023-07-12 02:53:53,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_96b5253932542ff8e0ec.png with policy now +2023-07-12 02:53:53,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_5420d62f64c76dfb3aaf.png with policy now +2023-07-12 02:53:53,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_ee4486348a50df1eb1b0.png with policy now +2023-07-12 02:53:53,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:53:53,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:53:53,466 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:53:53,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:53:53,835 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0btltlye-media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 02:53:53,842 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m1pyrm7a-media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 02:53:53,847 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rtmwrw25-media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 02:53:53,877 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hx3cvgrt-media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 02:53:56,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:53:56,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 02:53:56,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 02:53:56,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 02:53:56,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 02:53:56,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:53:56,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:56,724 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6fa02k7-media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 02:53:56,733 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/394ck36j-media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 02:53:56,746 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dil9q5xx-media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 02:53:56,795 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kpid1dzq-media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 02:53:59,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:02,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:02,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:03,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:03,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:03,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:05,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:07,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:08,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:08,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:54:11,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:12,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:12,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:12,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:12,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:12,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:14,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:14,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:17,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:17,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:19,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:19,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:20,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:22,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:22,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:22,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:22,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:22,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:23,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:23,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:28,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:29,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:32,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:32,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:32,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:32,742 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:32,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:33,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:34,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:34,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:35,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:35,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:38,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:38,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:54:38,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:41,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:43,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:43,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:43,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:43,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:44,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:44,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:47,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:50,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:50,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:50,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:50,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:53,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:53,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:53,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:53,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:53,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:55,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:56,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:56,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:59,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:01,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:04,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:04,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:04,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:04,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:05,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:05,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:06,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:06,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:06,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:08,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:08,875 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:55:11,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:11,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:14,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:14,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:14,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:14,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:14,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:14,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:16,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:17,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:21,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:21,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:22,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:23,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:25,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:25,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:25,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:25,212 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:26,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:27,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:29,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:29,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:29,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:29,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:29,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:32,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:32,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:33,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:36,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:36,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:38,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:55:38,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:43,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:48,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:51,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:51,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:54,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:59,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:04,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:06,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:06,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:08,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:56:09,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:14,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:19,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:21,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:21,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:24,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,221 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_1ff9e60aee819ba10a7e.png with policy now +2023-07-12 02:56:24,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_8220342bb1673b65f66a.png with policy now +2023-07-12 02:56:24,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,241 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_9749b628dfc772a6cc47.png with policy now +2023-07-12 02:56:24,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_dd5fdf92beda60fe8a12.png with policy now +2023-07-12 02:56:24,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:24,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:56:24,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:56:24,293 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:56:24,595 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gwlpzdh2-media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 02:56:24,596 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w4bsddzq-media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 02:56:24,597 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iefdxjb3-media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 02:56:24,632 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zu5v4r6d-media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 02:56:25,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:26,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:56:26,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 02:56:26,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 02:56:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 02:56:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 02:56:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:56:27,279 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iwlxkjb5-media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 02:56:27,289 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ulndhypd-media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 02:56:27,324 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9nbebl6u-media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 02:56:27,334 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hdelggno-media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 02:56:29,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:30,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:32,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:34,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:35,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:36,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:36,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:36,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:38,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:56:38,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:41,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:41,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:43,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:43,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:56:43,395 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:56:43,397 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:56:44,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:56:47,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:47,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:50,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:51,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:51,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:53,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:53,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:53,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:56:53,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:56:53,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:56:53,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:56:53,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:58,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:59,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:03,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:03,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:03,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:03,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:03,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:06,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:06,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:06,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:06,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:08,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:08,879 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:57:09,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:12,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:13,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:13,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:13,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:13,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:13,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:15,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:15,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:18,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:19,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:21,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:21,628 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:24,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:24,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:24,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:24,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:24,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:24,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:27,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:27,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:30,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:30,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:33,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:35,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:35,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:35,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:35,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:36,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:36,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:36,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:37,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:37,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:38,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:57:41,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:42,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:45,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:45,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:45,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:45,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:45,422 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:47,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:48,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:48,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:51,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:52,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:52,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:52,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:54,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:55,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:55,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:55,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:55,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:57,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:57,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:00,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:58:00,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:58:00,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:58:00,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:58:00,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:58:00,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:58:03,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:58:03,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:07,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:07,704 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:08,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:08,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:58:13,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:18,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:22,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:22,892 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:23,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:28,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:33,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:38,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:38,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:38,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:58:39,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:44,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:49,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:53,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:53,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:54,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_002747928010272ad4e0.png with policy now +2023-07-12 02:58:54,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_f85b220903af16d48ab4.png with policy now +2023-07-12 02:58:54,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_ccf6b7bd526cf61a21d8.png with policy now +2023-07-12 02:58:54,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_11469b6c8fd1d388b8bd.png with policy now +2023-07-12 02:58:54,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:58:54,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:58:54,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:58:54,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:58:55,068 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2fkphk8-media/images/validation_700_002747928010272ad4e0.png +2023-07-12 02:58:55,118 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1rux600b-media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 02:58:55,128 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e5wmjqc7-media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 02:58:55,130 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tf4qihmw-media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 02:58:55,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:57,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:58:57,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 02:58:57,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 02:58:57,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_002747928010272ad4e0.png +2023-07-12 02:58:57,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 02:58:57,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:58:57,726 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ru4trg10-media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 02:58:57,731 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kf1jvpgj-media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 02:58:57,736 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uinki999-media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 02:58:57,757 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/92i7cdk4-media/images/validation_700_002747928010272ad4e0.png +2023-07-12 02:59:00,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:01,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:03,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:04,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:06,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:06,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:08,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:08,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:08,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:59:09,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:12,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:12,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:13,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:13,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:13,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:13,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:15,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:15,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:18,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:18,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:21,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:23,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:23,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:23,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:24,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:24,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:24,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:24,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:24,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:24,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:27,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:29,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:30,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:33,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:34,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:35,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:35,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:35,208 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:35,210 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:36,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:36,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:38,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:38,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:38,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:59:39,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:42,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:45,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:45,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:45,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:45,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:45,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:45,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:45,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:48,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:50,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:51,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:53,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:53,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:54,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:55,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:55,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:55,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:55,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:55,700 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:57,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:00,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:00,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:03,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:06,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:06,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:06,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:06,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:06,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:06,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:06,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:08,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:08,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:08,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:00:09,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:11,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:12,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:15,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:16,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:16,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:16,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:16,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:17,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:18,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:18,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:21,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:23,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:23,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:23,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:24,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:26,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:26,628 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:26,628 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:26,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:27,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:27,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:28,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:30,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:31,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:31,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:31,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:31,536 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:33,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:33,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:34,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:36,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:38,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:38,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:38,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:00:39,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:44,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:49,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:53,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:53,716 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:55,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:00,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:05,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:08,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:08,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:08,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:01:10,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:15,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:20,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:23,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:23,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:25,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,884 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_f8ad001e252a38c302da.png with policy now +2023-07-12 03:01:25,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_e29ac85551bb1427b956.png with policy now +2023-07-12 03:01:25,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:25,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_4e2183818f3e746eee9b.png with policy now +2023-07-12 03:01:25,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_42e1f742968073010893.png with policy now +2023-07-12 03:01:25,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:25,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:01:25,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:01:25,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:01:26,188 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wnnp05iy-media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 03:01:26,232 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/au0x0odg-media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 03:01:26,247 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x12ho4sq-media/images/validation_710_42e1f742968073010893.png +2023-07-12 03:01:26,260 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gk4wb1q5-media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 03:01:27,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:01:27,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 03:01:27,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 03:01:27,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_42e1f742968073010893.png +2023-07-12 03:01:27,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 03:01:27,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:01:28,326 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bep6zlhy-media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 03:01:28,327 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lonqkvdj-media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 03:01:28,339 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7w2zz6nr-media/images/validation_710_42e1f742968073010893.png +2023-07-12 03:01:28,353 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8gm4kpk7-media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 03:01:30,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:31,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:33,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:35,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:36,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:37,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:38,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:38,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:38,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:01:40,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:42,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:43,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:45,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:45,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:01:45,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:01:45,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:01:46,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:01:46,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:47,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:49,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:52,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:53,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:54,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:54,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:54,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:54,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:01:54,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:01:54,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:01:55,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:01:55,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:58,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:58,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:03,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:04,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:04,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:04,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:04,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:04,854 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:07,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:07,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:08,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:08,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:02:09,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:09,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:10,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:13,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:14,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:15,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:15,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:15,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:15,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:16,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:16,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:19,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:22,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:25,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:25,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:25,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:25,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:26,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:26,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:26,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:26,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:28,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:28,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:30,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:31,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:34,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:35,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:35,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:35,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:35,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:35,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:37,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:38,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:02:40,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:40,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:40,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:40,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:43,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:45,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:45,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:45,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:45,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:46,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:46,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:46,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:51,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:52,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:55,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:55,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:55,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:55,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:55,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:55,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:55,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:56,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:58,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:58,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:03:00,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:03:00,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:03:00,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:03:00,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:03:01,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:03:01,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:03:02,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:04,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:03:07,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:08,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:03:10,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:10,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:12,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:17,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:22,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:25,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:25,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:28,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:33,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:38,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:38,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:03:40,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:40,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:43,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:48,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:53,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:55,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_0ecdc87dea1892df4eaf.png with policy now +2023-07-12 03:03:55,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_9c940486a7ed24710766.png with policy now +2023-07-12 03:03:55,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_a812b1f440f7f52d6145.png with policy now +2023-07-12 03:03:55,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,051 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_dc11c1be8869686a04d9.png with policy now +2023-07-12 03:03:55,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:03:55,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:03:55,098 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:03:55,104 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:03:55,332 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/loz4wj89-media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 03:03:55,334 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7nmp9aqn-media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 03:03:55,355 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aph4guql-media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 03:03:55,363 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ryrtxmhf-media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 03:03:55,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:03:55,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 03:03:55,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 03:03:55,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 03:03:55,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 03:03:55,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:03:55,805 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgyg7s1u-media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 03:03:55,829 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d4qwpi8k-media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 03:03:55,844 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7wm9c9vv-media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 03:03:55,871 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bfcgc6wp-media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 03:03:55,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:55,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:59,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:01,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:04,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:05,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:05,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:07,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:08,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:04:10,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:10,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:12,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:12,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:13,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:15,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:15,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:15,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:15,422 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:16,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:16,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:16,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:19,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:22,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:22,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:25,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:25,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:25,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:25,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:25,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:27,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:27,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:27,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:28,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:28,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:31,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:32,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:34,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:36,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:36,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:36,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:36,192 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:37,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:38,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:38,896 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:04:40,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:43,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:43,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:43,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:43,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:46,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:46,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:46,201 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:46,203 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:46,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:46,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:48,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:49,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:52,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:54,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:55,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:55,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:55,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:55,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:55,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:58,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:58,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:58,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:58,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:59,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:01,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:04,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:04,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:06,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:06,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:06,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:06,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:07,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:08,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:05:09,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:10,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:13,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:13,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:05:15,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:16,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:17,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:17,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:17,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:17,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:19,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:19,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:21,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:22,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:25,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:26,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:27,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:27,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:27,260 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:27,262 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:28,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:28,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:29,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:29,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:05:31,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:31,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:31,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:31,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:31,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:31,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:34,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:34,780 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:37,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:38,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:05:42,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:44,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:44,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:05:48,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:53,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:58,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:59,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:59,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:03,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:08,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:08,900 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:06:13,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:14,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:14,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:19,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:24,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:27,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_61523736602b93efd4f1.png with policy now +2023-07-12 03:06:27,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_b3042d96cd6c0051282b.png with policy now +2023-07-12 03:06:27,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_092236683fae816d1e06.png with policy now +2023-07-12 03:06:27,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_0d352eba665388c33ae3.png with policy now +2023-07-12 03:06:27,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:27,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:06:27,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:06:27,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:06:27,493 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cefn0ufl-media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 03:06:27,509 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/czuh5p18-media/images/validation_730_092236683fae816d1e06.png +2023-07-12 03:06:27,529 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b8avwlpz-media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 03:06:27,537 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7hzstif6-media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 03:06:28,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:06:28,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_092236683fae816d1e06.png +2023-07-12 03:06:28,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 03:06:28,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 03:06:28,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 03:06:28,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:06:29,327 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m9mpcfzu-media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 03:06:29,359 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ogyak5wo-media/images/validation_730_092236683fae816d1e06.png +2023-07-12 03:06:29,362 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ckyzeeii-media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 03:06:29,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:29,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:29,437 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/56e3hnkk-media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 03:06:29,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:31,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:34,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:37,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:38,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:38,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:06:40,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:41,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:44,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:44,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:44,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:46,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:46,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:46,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:06:46,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:06:46,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:06:47,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:06:47,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:50,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:52,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:53,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:56,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:56,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:56,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:06:56,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:06:56,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:06:58,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:59,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:06:59,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:59,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:59,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:02,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:03,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:05,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:06,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:06,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:06,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:06,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:08,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:08,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:07:11,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:13,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:14,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:14,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:07:14,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:16,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:16,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:16,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:16,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:17,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:17,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:18,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:20,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:23,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:23,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:26,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:26,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:26,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:26,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:26,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:29,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:29,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:29,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:29,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:07:29,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:32,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:34,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:35,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:36,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:36,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:36,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:36,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:38,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:38,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:38,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:07:39,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:41,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:44,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:44,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:07:44,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:45,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:47,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:47,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:47,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:47,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:47,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:50,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:50,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:50,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:53,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:56,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:56,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:58,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:58,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:58,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:58,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:59,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:59,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:08:00,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:00,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:02,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:02,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:08:03,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:08:03,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:08:03,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:08:03,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:08:05,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:08:05,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:08:07,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:08,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:08:12,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:15,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:15,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:17,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:22,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:27,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:30,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:30,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:33,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:38,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:08:39,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:44,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:45,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:45,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:49,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:54,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:58,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_4ca01dfc0ecd320c19a8.png with policy now +2023-07-12 03:08:58,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_472a400ca408d21e03bd.png with policy now +2023-07-12 03:08:58,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_d97239257eb1f6f99304.png with policy now +2023-07-12 03:08:58,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_46e887ae6b4e77b77e34.png with policy now +2023-07-12 03:08:58,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:08:58,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:08:58,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:08:58,173 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:08:58,457 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9r2jdwq0-media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 03:08:58,520 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1y09fqz2-media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 03:08:58,528 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1xi6qdo-media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 03:08:58,542 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3kb0h3xg-media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 03:08:59,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:08:59,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 03:08:59,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 03:08:59,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 03:08:59,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 03:08:59,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:08:59,843 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icmv2abo-media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 03:08:59,844 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjftq3t5-media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 03:08:59,865 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9bchwpsf-media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 03:08:59,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/84pgvy6x-media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 03:09:00,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:00,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:00,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:05,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:06,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:07,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:08,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:09:10,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:11,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:14,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:16,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:16,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:16,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:17,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:17,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:17,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:17,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:18,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:18,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:22,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:22,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:26,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:27,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:27,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:27,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:27,273 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:27,275 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:30,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:30,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:31,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:31,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:32,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:34,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:36,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:36,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:36,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:36,625 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:38,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:38,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:38,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:38,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:09:42,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:43,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:46,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:46,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:46,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:47,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:47,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:47,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:47,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:49,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:50,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:50,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:54,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:54,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:57,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:57,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:57,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:57,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:58,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:58,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:59,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:01,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:01,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:02,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:04,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:06,674 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:07,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:07,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:07,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:07,576 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:08,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:10:09,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:10,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:10,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:14,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:15,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:16,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:16,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:18,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:18,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:18,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:18,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:18,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:20,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:22,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:22,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:26,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:26,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:28,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:28,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:28,993 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:28,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:30,731 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:30,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:31,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:31,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:31,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:33,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:33,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:33,587 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:33,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:34,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:34,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:37,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:38,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:38,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:10:42,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:46,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:46,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:48,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:53,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:58,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:01,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:01,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:03,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:08,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:11:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:14,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:16,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:16,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:19,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:24,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:28,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_6a4042dd0f145a5a90ed.png with policy now +2023-07-12 03:11:28,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_d0782df35bd2b2c3cded.png with policy now +2023-07-12 03:11:28,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_f007accf327156196715.png with policy now +2023-07-12 03:11:28,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,123 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_952512d46b1737ec3172.png with policy now +2023-07-12 03:11:28,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:28,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:11:28,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:11:28,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:11:28,380 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pzh3jjk9-media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 03:11:28,442 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mpss25h0-media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 03:11:28,465 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oyxo5jr4-media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 03:11:28,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zc5qvpmt-media/images/validation_750_f007accf327156196715.png +2023-07-12 03:11:30,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:30,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:11:30,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 03:11:30,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 03:11:30,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 03:11:30,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_f007accf327156196715.png +2023-07-12 03:11:30,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:11:31,265 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kke9egdl-media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 03:11:31,273 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mep966ea-media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 03:11:31,274 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1lscod27-media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 03:11:31,301 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/68vwtfl5-media/images/validation_750_f007accf327156196715.png +2023-07-12 03:11:31,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:31,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:34,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:35,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:38,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:38,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:11:38,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:41,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:42,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:46,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:46,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:46,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:46,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:47,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:47,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:11:47,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:11:47,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:11:50,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:11:50,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:52,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:54,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:57,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:57,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:57,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:11:57,201 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:11:57,203 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:11:58,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:11:58,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:01,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:01,887 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:02,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:02,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:06,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:07,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:07,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:07,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:07,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:07,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:08,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:12:10,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:10,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:13,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:15,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:16,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:16,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:17,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:17,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:17,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:17,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:19,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:19,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:19,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:23,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:24,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:27,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:27,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:27,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:27,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:27,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:29,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:31,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:31,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:32,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:32,159 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:35,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:35,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:37,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:37,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:37,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:37,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:38,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:12:39,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:39,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:41,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:43,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:47,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:47,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:48,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:48,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:49,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:49,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:49,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:49,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:51,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:51,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:52,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:55,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:58,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:59,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:59,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:59,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:59,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:59,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:13:03,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:13:03,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:13:03,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:03,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:03,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:04,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:13:04,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:13:04,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:13:04,042 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:13:07,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:13:07,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:13:08,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:08,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:13:13,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:18,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:19,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:24,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:29,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:33,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:33,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:34,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:38,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:13:39,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:44,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:48,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:48,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:49,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:54,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:58,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_34b823a11a0cf4013af4.png with policy now +2023-07-12 03:13:58,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,354 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_af5422b14a97cc645625.png with policy now +2023-07-12 03:13:58,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,354 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_95d5c0a5c24789a3a475.png with policy now +2023-07-12 03:13:58,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_00dc8fdce08e1ae13772.png with policy now +2023-07-12 03:13:58,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:13:58,379 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:13:58,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:13:58,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:13:58,639 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gubehukp-media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 03:13:58,678 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rr2150nl-media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 03:13:58,693 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mrz4vr5o-media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 03:13:58,702 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8lz8sef9-media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 03:13:59,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:13:59,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 03:13:59,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 03:13:59,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 03:13:59,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 03:13:59,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:13:59,672 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/97ygv7ud-media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 03:13:59,673 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xlu2o9uc-media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 03:13:59,697 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mg6s6h5q-media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 03:13:59,790 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ef50zv5i-media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 03:14:00,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:03,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:03,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:03,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:05,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:07,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:07,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:08,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:14:11,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:11,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:15,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:17,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:17,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:17,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:17,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:17,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:19,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:19,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:19,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:19,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:22,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:23,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:27,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:27,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:27,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:27,260 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:27,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:27,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:27,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:31,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:32,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:35,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:35,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:35,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:37,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:38,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:38,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:38,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:38,196 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:38,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:14:39,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:39,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:43,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:43,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:47,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:47,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:47,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:47,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:47,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:49,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:50,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:50,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:51,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:51,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:54,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:55,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:58,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:58,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:58,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:58,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:59,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:59,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:00,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:03,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:05,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:06,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:06,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:07,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:08,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:15:09,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:09,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:09,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:09,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:11,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:11,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:11,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:15,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:16,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:19,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:19,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:19,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:19,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:19,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:19,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:21,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:21,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:21,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:23,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:26,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:27,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:29,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:29,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:29,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:29,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:31,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:31,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:32,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:34,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:34,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:34,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:34,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:35,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:35,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:36,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:36,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:39,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:15:39,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:44,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:49,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:51,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:51,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:55,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:00,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:05,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:06,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:06,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:08,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:16:10,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:15,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:20,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:21,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:21,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:28,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,465 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_df68bb8ae7cb3ad86921.png with policy now +2023-07-12 03:16:28,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png with policy now +2023-07-12 03:16:28,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_cba209758498588b8517.png with policy now +2023-07-12 03:16:28,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_217409d7c0158fc1dcec.png with policy now +2023-07-12 03:16:28,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:28,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:16:28,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:16:28,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:16:28,771 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cykuedw8-media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 03:16:28,813 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ijcx89sw-media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 03:16:28,834 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eylk86jg-media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 03:16:28,864 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3lcd60q-media/images/validation_770_cba209758498588b8517.png +2023-07-12 03:16:31,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:31,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:16:31,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 03:16:31,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_cba209758498588b8517.png +2023-07-12 03:16:31,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 03:16:31,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 03:16:31,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:16:32,157 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m77gpaq5-media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 03:16:32,163 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ek1jd4lq-media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 03:16:32,164 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r38pt0rd-media/images/validation_770_cba209758498588b8517.png +2023-07-12 03:16:32,166 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbl9urib-media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 03:16:35,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:36,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:36,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:36,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:38,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:16:39,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:39,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:42,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:43,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:47,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:48,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:48,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:48,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:16:48,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:16:48,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:16:51,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:51,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:51,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:16:51,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:53,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:55,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:58,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:58,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:16:58,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:16:58,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:16:59,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:59,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:16:59,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:03,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:04,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:07,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:07,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:07,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:07,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:07,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:07,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:07,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:07,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:08,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:17:09,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:11,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:14,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:15,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:17,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:17,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:17,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:17,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:19,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:19,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:20,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:22,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:22,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:23,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:26,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:27,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:27,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:27,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:27,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:27,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:27,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:31,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:31,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:35,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:36,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:37,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:37,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:37,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:37,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:37,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:37,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:38,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:17:39,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:39,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:41,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:43,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:47,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:47,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:49,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:49,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:49,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:49,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:52,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:52,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:52,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:52,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:52,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:56,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:57,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:59,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:59,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:59,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:59,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:18:00,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:18:00,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:18:03,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:04,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:18:04,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:18:04,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:18:04,525 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:18:04,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:18:07,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:07,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:08,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:18:08,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:18:08,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:18:08,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:13,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:18,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:22,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:22,503 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:24,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:29,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:34,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:37,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:37,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:38,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:18:39,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:44,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:49,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:52,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:52,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:55,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:58,729 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,729 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_5bc6ddbcab2869abe354.png with policy now +2023-07-12 03:18:58,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,769 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_8cd3d57f7ce32e8a2128.png with policy now +2023-07-12 03:18:58,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_8cb1b948d1d2b956831d.png with policy now +2023-07-12 03:18:58,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_6240b2c0fb96b2b24cb6.png with policy now +2023-07-12 03:18:58,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:18:58,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:18:58,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:18:58,805 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:18:59,080 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rddwit8k-media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 03:18:59,081 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qrgv0lb9-media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 03:18:59,090 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cbs7v4hx-media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 03:18:59,095 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bpg9h7d5-media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 03:19:00,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:00,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 03:19:00,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 03:19:00,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 03:19:00,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 03:19:00,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:19:00,568 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7lhv85g4-media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 03:19:00,578 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0c51lmqb-media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 03:19:00,586 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02qntig4-media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 03:19:00,604 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/umgsg1je-media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 03:19:00,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:04,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:06,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:07,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:07,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:08,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:08,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:19:08,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:12,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:12,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:16,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:17,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:17,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:17,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:17,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:17,903 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:20,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:20,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:22,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:22,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:23,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:24,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:27,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:27,833 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:27,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:27,835 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:28,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:28,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:28,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:32,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:34,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:36,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:37,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:37,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:38,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:38,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:38,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:38,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:38,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:19:39,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:40,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:40,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:44,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:48,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:49,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:49,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:49,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:49,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:51,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:52,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:52,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:52,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:52,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:56,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:56,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:00,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:00,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:00,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:00,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:00,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:00,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:02,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:04,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:07,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:07,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:07,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:08,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:08,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:20:10,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:10,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:10,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:10,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:12,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:12,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:12,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:16,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:17,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:20,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:21,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:21,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:21,502 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:21,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:23,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:23,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:23,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:24,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:24,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:28,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:28,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:31,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:31,861 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:31,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:31,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:32,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:32,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:33,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:36,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:36,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:36,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:36,374 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:36,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:36,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:38,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:38,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:38,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:20:38,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:40,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:43,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:48,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:53,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:53,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:54,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:59,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:04,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:08,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:08,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:08,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:21:09,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:14,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:19,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:23,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:23,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:25,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:30,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:30,832 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_f18cdbdebab2bd2197d4.png with policy now +2023-07-12 03:21:30,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_7c032ae6c9675c498efb.png with policy now +2023-07-12 03:21:30,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_0e3bece4b24d9f0cff63.png with policy now +2023-07-12 03:21:30,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_2c76661900a84617ca31.png with policy now +2023-07-12 03:21:30,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:21:30,907 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:21:30,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:21:30,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:21:31,135 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wohrxp4c-media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 03:21:31,166 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ma4qfgwq-media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 03:21:31,170 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ldbwo4n9-media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 03:21:31,187 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4bxfx133-media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 03:21:32,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:21:32,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 03:21:32,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 03:21:32,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 03:21:32,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 03:21:32,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:21:33,058 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhzt10zx-media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 03:21:33,074 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dmamb09u-media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 03:21:33,096 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ubiggxtu-media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 03:21:33,112 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q7ty2fyh-media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 03:21:35,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:36,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:38,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:38,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:38,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:21:40,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:40,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:41,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:21:44,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:46,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:48,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:50,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:21:50,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:21:50,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:21:50,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:21:52,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:52,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:21:52,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:53,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:53,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:56,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:57,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:00,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:00,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:00,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:00,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:00,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:00,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:02,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:04,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:07,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:08,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:08,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:08,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:08,935 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:22:11,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:11,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:11,118 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:11,120 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:12,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:12,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:13,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:16,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:18,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:20,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:21,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:21,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:21,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:21,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:23,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:23,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:23,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:24,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:24,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:28,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:28,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:31,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:31,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:31,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:31,229 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:32,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:32,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:34,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:36,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:38,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:38,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:38,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:22:39,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:41,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:42,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:42,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:42,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:42,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:45,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:45,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:45,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:49,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:50,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:52,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:52,977 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:52,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:52,979 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:53,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:53,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:53,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:53,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:56,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:57,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:01,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:01,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:02,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:23:02,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:23:02,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:23:02,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:23:05,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:23:05,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:07,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:07,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:23:07,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:23:07,275 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:23:07,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:23:08,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:08,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:08,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:23:09,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:23:09,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:12,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:13,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:17,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:22,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:23,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:23,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:28,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:33,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:38,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:38,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:38,579 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:38,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:23:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:48,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:53,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:53,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:54,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:59,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:01,323 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_a66b82318049cd1e171c.png with policy now +2023-07-12 03:24:01,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_ab1fea6190aed6bcea40.png with policy now +2023-07-12 03:24:01,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,366 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_c3dd46dd047d49388d6c.png with policy now +2023-07-12 03:24:01,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_9c63a115f12e85651901.png with policy now +2023-07-12 03:24:01,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:01,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:01,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:01,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:01,643 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y86rhgl1-media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 03:24:01,682 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/thtiht63-media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 03:24:01,697 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2ln7hpi-media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 03:24:01,701 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7x7gie5-media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 03:24:05,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 03:24:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 03:24:05,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 03:24:05,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 03:24:05,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:24:05,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:05,616 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nhvc10yb-media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 03:24:05,630 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0glcmc9z-media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 03:24:05,638 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8h0rlkvj-media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 03:24:05,648 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qq84huyp-media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 03:24:08,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:08,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:08,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:24:09,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:11,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:11,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:13,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:16,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:17,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:21,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:21,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:22,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:22,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:22,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:22,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:23,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:23,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:25,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:25,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:27,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:29,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:32,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:32,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:32,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:32,076 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:33,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:33,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:33,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:37,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:38,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:38,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:39,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:24:39,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:41,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:41,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:41,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:41,788 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:41,791 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:45,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:45,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:45,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:49,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:50,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:51,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:51,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:51,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:51,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:53,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:53,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:53,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:53,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:55,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:57,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:00,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:01,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:01,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:01,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:01,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:01,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:05,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:05,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:06,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:08,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:08,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:08,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:25:09,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:11,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:11,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:11,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:11,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:11,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:13,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:13,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:16,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:17,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:21,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:21,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:21,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:21,996 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:21,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:21,999 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:23,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:23,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:25,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:25,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:27,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:29,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:32,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:32,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:32,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:32,788 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:32,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:33,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:33,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:37,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:37,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:37,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:37,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:37,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:38,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:38,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:25:38,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:38,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:41,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:41,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:44,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:53,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:53,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:54,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:00,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:08,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:26:08,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:08,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:10,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:15,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:20,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:23,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:23,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:25,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:30,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:32,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_9a62636574ba68d8fc83.png with policy now +2023-07-12 03:26:32,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_8d25802cfd68926c4540.png with policy now +2023-07-12 03:26:32,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_14a8076e3481bc7a8090.png with policy now +2023-07-12 03:26:32,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_3d586403ca68c8dd89ef.png with policy now +2023-07-12 03:26:32,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:26:32,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:26:32,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:26:32,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:26:32,455 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s9t517t5-media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 03:26:32,473 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7c6198ro-media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 03:26:32,483 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oio1uc1i-media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 03:26:32,495 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lccyq7ou-media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 03:26:33,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:26:33,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 03:26:33,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 03:26:33,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 03:26:33,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 03:26:33,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:26:34,031 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tep2ovo9-media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 03:26:34,048 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lmqxju77-media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 03:26:34,049 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0c5xolym-media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 03:26:34,058 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pclu2lri-media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 03:26:35,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:37,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:38,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:26:38,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:38,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:40,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:41,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:42,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:26:45,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:45,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:49,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:50,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:52,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:26:52,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:26:52,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:26:52,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:26:53,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:26:53,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:53,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:53,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:56,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:57,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:01,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:01,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:02,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:02,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:02,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:02,200 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:05,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:05,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:07,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:08,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:27:09,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:09,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:09,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:12,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:12,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:12,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:12,750 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:12,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:13,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:13,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:17,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:18,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:21,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:22,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:22,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:22,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:22,479 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:24,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:24,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:24,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:25,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:25,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:29,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:29,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:32,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:32,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:32,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:32,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:33,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:33,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:34,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:37,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:38,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:27:39,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:39,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:41,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:41,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:42,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:42,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:42,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:42,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:45,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:45,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:46,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:49,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:52,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:53,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:53,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:53,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:53,136 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:53,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:53,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:55,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:55,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:57,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:57,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:01,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:03,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:28:03,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:03,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:28:03,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:28:03,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:28:05,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:28:05,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:08,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:28:08,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:28:08,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:28:08,177 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:28:08,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:08,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:28:09,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:28:09,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:10,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:10,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:13,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:14,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:19,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:24,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:25,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:25,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:29,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:34,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:38,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:28:39,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:40,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:40,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:45,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:50,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:55,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:55,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:56,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:01,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:02,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_9eeeace7edf859719f14.png with policy now +2023-07-12 03:29:02,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,705 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_498f985682f98f0b515d.png with policy now +2023-07-12 03:29:02,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_c37a81321646db9654fb.png with policy now +2023-07-12 03:29:02,716 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_de63bd88f0f3c4502afa.png with policy now +2023-07-12 03:29:02,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:02,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:02,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:02,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:02,981 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yygxsyx0-media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 03:29:03,028 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/akju31dt-media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 03:29:03,068 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rhbf86er-media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 03:29:03,127 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8ee01sa-media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 03:29:06,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:06,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 03:29:06,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 03:29:06,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 03:29:06,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 03:29:06,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:29:06,523 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y08j9pma-media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 03:29:06,536 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pgtr06pp-media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 03:29:06,553 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/048c6o36-media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 03:29:06,561 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k3owf84e-media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 03:29:06,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:08,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:29:10,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:10,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:10,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:12,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:13,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:14,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:17,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:18,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:22,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:22,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:22,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:22,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:22,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:23,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:25,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:25,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:26,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:26,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:29,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:30,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:31,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:31,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:31,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:31,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:34,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:34,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:34,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:38,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:38,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:29:40,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:40,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:40,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:41,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:41,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:41,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:41,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:42,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:42,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:45,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:46,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:50,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:50,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:52,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:52,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:52,193 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:52,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:54,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:54,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:55,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:55,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:56,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:58,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:02,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:02,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:02,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:02,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:02,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:02,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:06,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:06,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:07,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:08,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:30:10,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:10,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:10,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:12,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:12,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:12,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:12,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:13,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:14,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:14,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:18,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:19,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:22,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:22,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:22,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:22,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:22,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:24,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:25,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:25,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:26,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:26,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:30,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:30,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:33,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:33,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:33,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:33,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:34,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:34,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:35,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:38,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:38,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:38,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:38,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:38,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:38,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:30:40,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:40,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:41,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:42,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:42,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:46,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:51,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:55,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:55,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:56,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:01,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:06,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:08,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:31:10,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:10,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:11,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:16,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:21,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:25,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:25,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:26,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:31,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:33,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_5d081a2a0528a30e8ce2.png with policy now +2023-07-12 03:31:33,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_0aca2d7047838c676be5.png with policy now +2023-07-12 03:31:33,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_3bdbbfd9d4a0076d30db.png with policy now +2023-07-12 03:31:33,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_73577e16f9e832867bc6.png with policy now +2023-07-12 03:31:33,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:31:33,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:31:33,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:31:33,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:31:33,725 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m8sesui5-media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 03:31:34,238 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/krcz232f-media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 03:31:34,324 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n7vs1mha-media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 03:31:34,443 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fndsi8k1-media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 03:31:34,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:31:34,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 03:31:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 03:31:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 03:31:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 03:31:34,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:31:35,024 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j0hkuzp1-media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 03:31:35,068 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xq54jwsm-media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 03:31:35,071 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k4e0ddv0-media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 03:31:35,460 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kksr49tt-media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 03:31:36,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:38,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:38,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:31:41,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:41,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:42,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:42,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:43,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:31:46,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:47,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:50,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:53,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:31:53,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:53,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:31:53,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:31:53,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:31:54,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:31:54,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:56,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:56,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:58,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:58,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:02,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:02,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:02,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:02,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:03,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:06,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:08,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:32:08,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:10,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:11,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:11,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:13,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:13,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:13,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:13,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:14,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:14,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:14,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:18,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:20,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:22,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:24,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:24,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:24,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:24,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:25,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:26,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:26,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:26,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:26,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:30,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:31,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:34,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:34,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:34,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:34,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:34,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:34,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:36,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:38,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:38,959 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:32:41,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:41,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:42,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:42,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:44,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:44,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:44,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:44,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:46,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:46,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:47,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:50,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:53,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:54,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:56,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:56,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:56,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:56,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:56,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:58,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:58,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:58,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:02,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:04,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:06,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:33:06,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:33:06,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:33:06,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:33:06,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:33:06,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:08,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:33:09,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:10,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:33:10,716 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:33:10,716 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:33:10,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:33:10,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:33:10,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:11,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:11,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:33:14,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:15,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:20,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:25,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:26,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:26,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:33:30,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:35,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:38,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:33:40,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:41,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:41,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:33:45,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:50,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:55,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:57,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:57,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:01,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:05,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_90e1e0a7b7d300559299.png with policy now +2023-07-12 03:34:05,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_3cc72b352f6a24b1d731.png with policy now +2023-07-12 03:34:05,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_924f6a01834fef3e5df9.png with policy now +2023-07-12 03:34:05,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,377 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_8376c90a668eab0df0f1.png with policy now +2023-07-12 03:34:05,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:05,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:05,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:05,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:05,658 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bu54zc7x-media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 03:34:05,666 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zc6cgqp9-media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 03:34:05,676 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gbnuoeu3-media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 03:34:05,690 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s9i8iiyj-media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 03:34:06,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:07,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 03:34:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 03:34:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 03:34:07,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 03:34:07,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:34:07,527 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sfsptab2-media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 03:34:07,528 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mmrqxjlv-media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 03:34:07,538 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4anwj15k-media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 03:34:07,540 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rt1ipyit-media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 03:34:08,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:34:11,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:12,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:12,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:12,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:15,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:15,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:17,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:19,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:23,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:23,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:24,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:24,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:24,249 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:24,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:27,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:27,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:28,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:28,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:28,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:31,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:33,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:33,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:33,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:33,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:34,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:35,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:35,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:38,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:34:39,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:39,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:43,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:43,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:43,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:43,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:43,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:43,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:43,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:45,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:47,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:47,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:50,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:51,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:53,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:53,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:53,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:53,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:55,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:55,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:55,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:58,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:58,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:59,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:00,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:03,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:03,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:03,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:03,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:03,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:06,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:07,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:07,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:08,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:35:11,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:11,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:13,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:13,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:35:14,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:14,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:14,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:14,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:15,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:15,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:16,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:19,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:22,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:23,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:25,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:25,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:25,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:25,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:27,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:27,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:27,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:29,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:29,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:35:31,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:33,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:34,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:34,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:34,982 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:34,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:35,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:35,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:38,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:35:38,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:39,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:39,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:39,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:39,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:39,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:43,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:43,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:44,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:44,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:44,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:35:49,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:54,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:59,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:59,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:59,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:04,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:08,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:36:09,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:14,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:14,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:15,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:20,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:25,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:29,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:29,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:30,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:34,440 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,440 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_d8f23be87659ed0cbfb4.png with policy now +2023-07-12 03:36:34,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_20f5209698af0ab0b778.png with policy now +2023-07-12 03:36:34,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_20c9060ffc3bdad4a934.png with policy now +2023-07-12 03:36:34,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_1f72867e944a10d3e520.png with policy now +2023-07-12 03:36:34,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:36:34,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:36:34,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:36:34,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:36:34,865 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/79x5z2yb-media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 03:36:34,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gpozjq6b-media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 03:36:34,902 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nnu0aab8-media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 03:36:34,934 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7i5zwlkv-media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 03:36:35,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:36:35,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 03:36:35,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 03:36:35,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 03:36:35,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 03:36:35,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:36:36,120 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x8h7sd23-media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 03:36:36,137 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uxacb6z3-media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 03:36:36,161 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6xskmv1h-media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 03:36:36,193 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wji6g6tp-media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 03:36:36,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:38,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:36:39,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:42,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:43,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:44,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:36:44,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:44,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:47,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:47,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:51,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:53,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:54,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:36:54,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:36:54,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:36:54,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:36:55,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:36:55,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:58,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:59,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:59,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:59,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:03,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:03,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:04,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:04,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:04,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:04,491 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:07,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:07,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:08,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:37:08,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:11,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:14,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:14,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:37:14,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:14,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:14,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:14,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:14,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:15,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:15,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:19,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:19,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:23,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:25,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:25,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:25,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:25,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:25,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:27,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:27,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:29,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:37:29,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:30,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:31,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:35,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:35,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:35,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:35,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:35,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:35,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:35,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:38,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:37:39,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:41,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:43,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:44,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:37:44,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:45,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:45,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:45,445 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:45,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:46,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:47,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:47,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:51,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:51,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:55,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:55,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:55,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:55,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:55,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:55,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:57,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:59,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:00,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:00,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:03,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:03,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:05,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:38:05,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:38:05,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:38:05,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:38:07,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:38:07,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:08,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:38:08,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:10,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:38:10,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:38:10,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:38:10,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:38:11,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:38:11,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:14,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:15,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:15,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:20,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:25,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:30,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:30,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:31,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:36,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:38,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:38:41,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:45,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:45,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:47,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:52,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:57,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:00,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:00,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:03,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:04,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_97fe22a2e02d107e0669.png with policy now +2023-07-12 03:39:04,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,619 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_8062bd4106d50ccb836d.png with policy now +2023-07-12 03:39:04,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_578c282df31c9a089068.png with policy now +2023-07-12 03:39:04,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_1ff63456305716e15a0f.png with policy now +2023-07-12 03:39:04,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:04,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:04,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:04,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:04,967 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1s2mjyqo-media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 03:39:04,980 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/imalsj4f-media/images/validation_860_578c282df31c9a089068.png +2023-07-12 03:39:05,020 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/04tfg4q6-media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 03:39:05,086 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ccm5qmt3-media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 03:39:08,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:08,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:08,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 03:39:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 03:39:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_578c282df31c9a089068.png +2023-07-12 03:39:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 03:39:08,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:39:08,556 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v6pyph4t-media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 03:39:08,633 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5l3ofxxi-media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 03:39:08,642 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rzrj8et1-media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 03:39:08,689 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qkciqu03-media/images/validation_860_578c282df31c9a089068.png +2023-07-12 03:39:08,976 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:39:08,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:12,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:14,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:14,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:15,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:15,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:16,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:19,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:20,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:23,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:23,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:23,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:23,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:24,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:24,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:24,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:28,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:30,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:30,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:30,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:32,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:33,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:33,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:33,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:33,044 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:35,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:36,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:36,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:38,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:39:40,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:40,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:43,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:43,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:43,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:43,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:44,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:44,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:45,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:45,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:46,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:48,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:52,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:52,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:53,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:53,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:53,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:53,552 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:56,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:56,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:57,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:00,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:01,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:01,232 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:02,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:04,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:04,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:04,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:04,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:04,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:08,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:08,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:08,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:08,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:40:12,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:13,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:14,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:14,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:14,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:14,567 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:16,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:16,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:16,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:16,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:18,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:20,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:24,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:24,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:24,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:24,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:24,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:24,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:28,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:28,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:29,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:32,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:32,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:32,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:34,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:34,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:34,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:34,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:34,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:36,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:36,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:38,981 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:40:39,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:40,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:40,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:40,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:40,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:40,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:40,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:44,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:45,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:47,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:47,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:51,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:56,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:01,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:02,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:02,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:06,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:08,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:41:11,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:16,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:17,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:17,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:22,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:27,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:32,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:32,283 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:32,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:34,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_b6004993162c340948e4.png with policy now +2023-07-12 03:41:34,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_1e7e11e8737344ca3c57.png with policy now +2023-07-12 03:41:34,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_4f7e5756a84f41b58f03.png with policy now +2023-07-12 03:41:34,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_674d99e976453deac9a0.png with policy now +2023-07-12 03:41:34,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:41:34,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:41:34,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:41:34,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:41:34,677 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yjhaez30-media/images/validation_870_b6004993162c340948e4.png +2023-07-12 03:41:34,738 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sincgc7j-media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 03:41:34,871 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pamxhpxv-media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 03:41:34,899 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t891uilu-media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 03:41:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:41:36,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 03:41:36,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 03:41:36,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 03:41:36,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_b6004993162c340948e4.png +2023-07-12 03:41:36,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:41:36,984 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oubo216i-media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 03:41:37,109 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5ebnwx9n-media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 03:41:37,164 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vgofr6ln-media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 03:41:37,169 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hm9uhm8s-media/images/validation_870_b6004993162c340948e4.png +2023-07-12 03:41:38,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:38,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:41:40,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:43,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:44,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:41:44,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:47,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:47,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:48,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:49,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:52,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:52,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:41:52,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:41:52,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:41:52,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:41:54,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:56,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:41:56,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:00,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:00,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:01,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:01,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:01,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:01,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:02,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:02,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:04,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:04,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:06,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:08,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:08,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:42:11,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:11,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:11,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:11,655 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:12,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:12,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:12,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:16,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:17,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:17,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:18,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:20,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:22,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:22,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:22,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:22,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:24,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:24,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:24,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:28,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:29,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:32,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:32,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:32,535 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:32,536 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:32,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:32,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:32,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:32,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:34,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:37,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:38,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:42:39,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:41,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:43,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:43,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:43,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:43,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:45,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:45,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:45,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:47,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:47,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:49,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:50,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:53,088 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:53,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:53,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:53,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:53,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:55,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:57,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:57,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:01,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:01,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:02,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:02,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:43:05,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:43:05,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:43:05,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:43:05,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:43:05,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:07,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:08,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:43:09,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:09,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:43:09,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:43:09,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:43:09,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:43:12,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:13,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:43:13,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:17,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:17,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:18,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:23,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:28,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:32,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:32,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:34,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:38,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:43:39,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:44,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:48,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:48,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:50,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:55,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:00,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:03,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:03,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:03,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_c05827ace7407dafe67f.png with policy now +2023-07-12 03:44:03,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_ddb13591ebb5fc4093e5.png with policy now +2023-07-12 03:44:03,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_1a93cee3c2fe9238634b.png with policy now +2023-07-12 03:44:03,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_8a9698cc7d3b8724fb2b.png with policy now +2023-07-12 03:44:03,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:03,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:03,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:03,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:03,867 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q3mam0su-media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 03:44:03,869 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bixqksfj-media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 03:44:03,885 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhot14m0-media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 03:44:03,909 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d5egy65t-media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 03:44:05,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:05,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 03:44:05,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 03:44:05,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 03:44:05,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 03:44:05,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:44:05,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:05,699 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/47kf4l6b-media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 03:44:05,702 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/32hzjj9w-media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 03:44:05,744 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gzkec77v-media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 03:44:05,812 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uvox1gun-media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 03:44:08,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:44:09,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:10,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:13,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:13,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:15,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:17,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:18,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:18,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:21,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:21,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:22,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:22,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:22,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:22,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:25,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:25,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:29,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:32,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:32,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:32,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:32,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:32,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:33,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:33,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:33,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:33,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:37,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:38,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:38,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:44:41,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:42,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:42,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:42,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:42,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:43,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:45,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:45,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:49,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:49,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:49,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:49,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:51,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:51,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:51,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:51,976 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:53,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:53,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:54,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:57,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:00,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:01,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:03,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:03,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:03,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:03,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:04,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:04,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:05,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:05,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:05,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:08,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:45:09,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:10,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:13,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:13,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:13,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:13,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:13,745 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:16,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:17,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:17,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:19,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:19,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:21,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:21,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:24,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:24,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:24,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:24,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:25,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:25,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:26,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:29,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:32,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:33,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:34,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:34,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:34,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:34,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:34,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:34,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:37,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:37,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:37,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:38,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:45:38,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:39,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:39,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:39,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:41,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:41,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:43,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:48,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:49,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:49,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:54,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:59,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:04,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:04,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:04,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:08,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:46:09,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:14,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:19,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:19,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:20,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:25,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:30,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:33,652 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_f18e48cd66c93df800cc.png with policy now +2023-07-12 03:46:33,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_5f8815ecaca9bc2b8ad7.png with policy now +2023-07-12 03:46:33,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,694 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_01b5da4e3887f97a0716.png with policy now +2023-07-12 03:46:33,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_e9f5b2cb5a42a51a191f.png with policy now +2023-07-12 03:46:33,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:46:33,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:46:33,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:46:33,725 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:46:33,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:46:33,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 03:46:33,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 03:46:33,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 03:46:33,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 03:46:33,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:46:34,032 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uttiziye-media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 03:46:34,037 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rvh68cir-media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 03:46:34,043 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w2jlwyia-media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 03:46:34,200 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e6g6006d-media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 03:46:34,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:34,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:34,512 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nwtg00jw-media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 03:46:34,578 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h3hbtcz2-media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 03:46:34,579 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tp57ewhw-media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 03:46:34,753 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/edzsiwpp-media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 03:46:35,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:38,996 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:46:40,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:41,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:43,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:46:45,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:46,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:49,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:49,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:49,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:51,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:53,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:46:53,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:46:53,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:46:53,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:46:53,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:46:53,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:57,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:57,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:01,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:02,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:02,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:02,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:02,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:02,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:04,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:04,563 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:05,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:05,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:07,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:08,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:47:09,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:13,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:13,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:13,364 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:13,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:13,366 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:13,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:13,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:17,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:19,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:20,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:20,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:21,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:24,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:24,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:24,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:24,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:24,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:25,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:25,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:29,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:29,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:34,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:34,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:34,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:34,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:34,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:34,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:35,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:35,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:38,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:38,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:38,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:47:40,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:42,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:44,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:44,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:44,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:44,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:46,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:46,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:46,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:50,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:51,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:51,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:51,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:54,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:54,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:54,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:54,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:54,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:56,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:58,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:58,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:01,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:02,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:05,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:48:05,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:48:05,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:48:05,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:48:06,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:48:06,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:07,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:07,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:07,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:09,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:48:09,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:48:09,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:48:09,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:48:09,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:48:10,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:48:10,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:12,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:14,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:17,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:22,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:22,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:23,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:28,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:33,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:37,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:37,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:39,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:48:39,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:44,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:49,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:52,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:52,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:54,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:59,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:03,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_6956b8efd11196c8a652.png with policy now +2023-07-12 03:49:03,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,846 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_5741bd49ef7880023ce2.png with policy now +2023-07-12 03:49:03,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,846 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_54eb3287687fb6f3d07f.png with policy now +2023-07-12 03:49:03,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_4bf5e195f1228d67ca30.png with policy now +2023-07-12 03:49:03,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:03,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:03,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:03,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:04,153 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wvkl55sr-media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 03:49:04,275 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhay3qdf-media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 03:49:04,350 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t40saclh-media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 03:49:04,437 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpvbueie-media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 03:49:04,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:06,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:06,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 03:49:06,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 03:49:06,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 03:49:06,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 03:49:06,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:49:06,647 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qwu9gztl-media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 03:49:06,711 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9gqscbkg-media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 03:49:06,791 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8pt5gvyt-media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 03:49:06,871 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lkhguiws-media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 03:49:07,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:07,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:09,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:49:10,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:10,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:14,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:14,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:16,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:18,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:21,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:22,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:22,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:23,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:23,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:23,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:23,517 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:26,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:26,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:27,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:30,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:32,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:34,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:34,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:34,032 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:34,034 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:34,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:34,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:38,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:38,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:38,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:38,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:39,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:49:42,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:43,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:44,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:44,028 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:44,028 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:44,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:46,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:46,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:49,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:50,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:53,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:53,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:54,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:54,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:54,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:54,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:54,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:54,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:58,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:58,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:59,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:02,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:04,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:04,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:04,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:04,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:05,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:06,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:06,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:09,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:50:09,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:09,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:10,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:10,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:14,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:15,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:15,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:15,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:15,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:16,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:18,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:18,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:21,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:22,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:24,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:24,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:25,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:25,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:25,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:25,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:26,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:26,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:26,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:30,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:32,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:34,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:35,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:35,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:35,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:35,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:37,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:38,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:38,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:39,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:50:39,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:39,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:39,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:39,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:39,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:39,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:42,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:42,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:43,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:48,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:53,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:54,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:54,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:59,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:04,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:09,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:51:09,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:09,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:10,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:15,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:20,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:25,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:25,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:25,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:30,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:33,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:33,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_f56c6743f3df48566c7e.png with policy now +2023-07-12 03:51:34,033 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:34,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_85b1581a7259fb0cc926.png with policy now +2023-07-12 03:51:34,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:34,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_4bf762954166a9bf2bce.png with policy now +2023-07-12 03:51:34,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:34,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_2e5e7f759b91d9be266a.png with policy now +2023-07-12 03:51:34,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:51:34,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:51:34,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:51:34,109 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:51:34,372 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hnfz50f2-media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 03:51:34,374 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w3dy1cfx-media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 03:51:34,420 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2p6rjvg-media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 03:51:34,445 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nohdn0w4-media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 03:51:34,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:51:34,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 03:51:34,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 03:51:34,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 03:51:34,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 03:51:34,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:51:35,237 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yu3i2tfe-media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 03:51:35,264 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j893bq0w-media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 03:51:35,364 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2mq97okc-media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 03:51:35,382 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hvc9t1eg-media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 03:51:36,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:38,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:39,008 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:51:40,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:40,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:41,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:42,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:44,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:51:46,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:46,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:50,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:52,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:54,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:51:54,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:51:54,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:51:54,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:51:54,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:51:54,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:55,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:55,731 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:57,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:02,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:02,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:04,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:04,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:04,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:04,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:06,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:06,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:07,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:09,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:52:10,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:11,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:11,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:13,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:14,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:14,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:14,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:14,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:14,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:14,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:18,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:18,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:22,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:24,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:25,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:25,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:25,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:25,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:26,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:26,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:26,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:26,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:29,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:30,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:34,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:35,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:35,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:35,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:35,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:35,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:39,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:52:39,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:39,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:40,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:42,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:42,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:43,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:45,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:45,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:45,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:45,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:45,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:47,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:47,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:50,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:51,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:55,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:56,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:56,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:56,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:56,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:56,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:58,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:58,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:59,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:59,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:01,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:03,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:05,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:53:05,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:53:05,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:53:05,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:53:07,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:53:07,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:07,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:09,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:53:10,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:53:10,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:53:10,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:53:10,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:53:11,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:53:11,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:12,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:13,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:13,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:53:15,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:18,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:23,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:28,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:28,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:53:28,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:33,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:38,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:39,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:53:43,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:43,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:43,286 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:53:48,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:53,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:58,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:58,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:58,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:03,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:04,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,264 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_3ac6586c86738660d7b9.png with policy now +2023-07-12 03:54:04,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_7f02fda774e0cc0ec92a.png with policy now +2023-07-12 03:54:04,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_8544a6b986b0e7758e17.png with policy now +2023-07-12 03:54:04,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_4cbb72cb2a336b2f4f2b.png with policy now +2023-07-12 03:54:04,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:04,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:04,336 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:04,338 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:04,610 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7528qevp-media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 03:54:04,613 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9q5k60s-media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 03:54:04,684 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s3g25wpf-media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 03:54:04,822 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u99d50nw-media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 03:54:07,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:07,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 03:54:07,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 03:54:07,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 03:54:07,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 03:54:07,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:54:07,701 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/97d34b2f-media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 03:54:07,710 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jg6jp207-media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 03:54:07,721 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8yot8tdx-media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 03:54:07,731 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b2hed0lj-media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 03:54:09,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:54:09,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:11,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:13,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:13,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:14,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:14,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:15,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:19,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:20,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:23,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:23,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:23,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:23,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:23,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:25,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:27,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:27,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:28,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:28,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:31,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:31,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:33,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:33,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:33,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:33,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:35,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:35,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:36,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:39,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:54:39,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:42,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:42,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:42,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:42,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:42,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:43,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:43,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:44,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:44,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:47,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:47,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:51,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:53,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:53,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:53,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:53,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:53,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:55,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:55,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:58,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:59,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:59,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:59,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:03,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:03,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:03,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:03,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:03,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:03,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:07,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:07,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:09,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:55:09,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:11,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:12,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:12,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:12,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:12,850 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:14,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:15,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:55:15,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:15,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:15,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:19,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:20,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:23,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:24,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:24,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:24,067 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:24,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:26,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:27,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:27,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:30,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:55:30,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:31,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:31,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:35,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:35,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:35,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:35,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:35,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:35,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:37,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:39,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:55:39,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:39,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:39,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:39,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:39,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:39,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:42,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:43,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:45,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:55:45,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:48,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:53,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:58,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:00,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:00,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:04,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:09,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:56:10,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:15,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:15,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:15,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:20,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:25,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:30,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:30,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:30,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:33,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,710 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_a2e24d69369ee9545e6c.png with policy now +2023-07-12 03:56:33,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_98329b1eb9c39ec50100.png with policy now +2023-07-12 03:56:33,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_14811891692332578df2.png with policy now +2023-07-12 03:56:33,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_708543de31a4012af277.png with policy now +2023-07-12 03:56:33,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:56:33,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:56:33,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:56:33,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:56:34,030 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l05wg23a-media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 03:56:34,102 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ef5snuc4-media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 03:56:34,109 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4m30hi2l-media/images/validation_930_708543de31a4012af277.png +2023-07-12 03:56:34,206 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9rwgd1u2-media/images/validation_930_14811891692332578df2.png +2023-07-12 03:56:35,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:35,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:56:35,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_14811891692332578df2.png +2023-07-12 03:56:35,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 03:56:35,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 03:56:35,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_708543de31a4012af277.png +2023-07-12 03:56:35,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:56:36,211 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9oz98uum-media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 03:56:36,211 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2r9zkass-media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 03:56:36,580 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ssho1h6j-media/images/validation_930_14811891692332578df2.png +2023-07-12 03:56:36,631 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8w9fmdub-media/images/validation_930_708543de31a4012af277.png +2023-07-12 03:56:39,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:56:39,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:41,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:43,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:56:43,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:45,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:45,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:46,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:47,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:51,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:51,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:52,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:56:52,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:56:52,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:56:52,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:56:55,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:56:55,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:57,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:59,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:00,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:00,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:03,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:06,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 03:57:10,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:10,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:10,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:10,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:10,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:57:11,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:11,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:12,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:12,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:12,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:12,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:15,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:15,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:15,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:15,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:15,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:19,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:20,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:21,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:21,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:21,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:21,916 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:23,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:23,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:26,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:27,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:30,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:30,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:31,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:31,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:32,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:32,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:32,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:32,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:36,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:36,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:36,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:39,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:57:40,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:41,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:42,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:42,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:42,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:42,873 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:44,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:44,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:45,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:45,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:46,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:48,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:51,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:52,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:53,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:53,376 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:53,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:53,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:56,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:56,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:57,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:00,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:00,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:00,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:02,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:03,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:58:03,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:58:03,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:58:03,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:58:04,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:58:04,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:08,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:58:08,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:08,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:58:08,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:58:08,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:58:08,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:58:08,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:09,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:58:12,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:13,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:15,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:15,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:18,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:23,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:28,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:30,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:30,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:33,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:38,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:39,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:58:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:45,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:45,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:49,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:54,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:59,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:00,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:00,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:02,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_1aa1dd5671e7761dc7f9.png with policy now +2023-07-12 03:59:02,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_25baa72fa29a6b87a7e3.png with policy now +2023-07-12 03:59:02,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_55fb249dfc64d037ba51.png with policy now +2023-07-12 03:59:02,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_0f4ce489e03b7c6499a4.png with policy now +2023-07-12 03:59:02,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:02,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:02,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:02,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:03,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jltbj46f-media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 03:59:03,030 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cai0s1j8-media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 03:59:03,148 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f8ifli6d-media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 03:59:03,212 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/md76v5eb-media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 03:59:04,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:04,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 03:59:04,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 03:59:04,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 03:59:04,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 03:59:04,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:59:04,708 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wpb60rw1-media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 03:59:04,732 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4iqlfmn9-media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 03:59:04,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:08,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:09,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:59:10,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:10,461 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zol84uvh-media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 03:59:12,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:13,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:15,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:15,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:15,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:16,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:17,257 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ae26ohgk-media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 03:59:20,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:20,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:23,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:23,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:23,179 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:23,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:24,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:24,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:26,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:28,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:30,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:30,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:31,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:32,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:33,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:33,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:33,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:33,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:36,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:36,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:37,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:39,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:59:40,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:42,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:43,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:43,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:43,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:43,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:44,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:44,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:45,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:45,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:47,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:48,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:52,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:53,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:53,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:53,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:53,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:53,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:56,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:56,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:58,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:00,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:00,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:00,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:03,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:03,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:03,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:03,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:03,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:04,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:04,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:08,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:08,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:09,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:00:12,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:13,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:14,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:14,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:14,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:14,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:16,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:16,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:16,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:16,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:19,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:20,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:24,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:24,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:24,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:24,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:24,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:24,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:24,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:28,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:30,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:31,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:31,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:32,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:34,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:34,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:34,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:34,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:36,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:36,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:36,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:39,028 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:00:39,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:39,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:39,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:39,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:40,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:40,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:42,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:44,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:46,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:46,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:47,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:52,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:57,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:01,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:01,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:02,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:07,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:09,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:01:13,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:16,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:18,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:23,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:28,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:31,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:31,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:34,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_8e55f5be94ca16a527e8.png with policy now +2023-07-12 04:01:34,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:34,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_cedfd2e3a8d728719314.png with policy now +2023-07-12 04:01:34,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,078 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_b66230be0915df8b41e3.png with policy now +2023-07-12 04:01:34,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_32af62f9d0e790c02608.png with policy now +2023-07-12 04:01:34,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:01:34,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:01:34,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:01:34,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:01:34,386 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hpqqjty1-media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 04:01:34,413 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q0zoi9sc-media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 04:01:34,453 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6u2cy8p-media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 04:01:34,592 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qcl017ms-media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 04:01:36,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:01:36,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 04:01:36,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 04:01:36,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 04:01:36,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 04:01:36,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:01:37,236 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5si1vp9u-media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 04:01:37,307 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5x3cd3de-media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 04:01:37,312 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uvinis4v-media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 04:01:37,351 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jsiqxi9o-media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 04:01:39,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:01:40,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:40,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:44,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:01:44,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:45,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:46,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:46,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:48,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:51,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:52,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:53,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:01:53,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:01:53,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:01:53,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:01:56,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:56,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:01:56,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:00,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:01,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:01,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:01,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:03,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:03,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:03,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:03,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:04,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:04,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:06,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:08,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:09,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:02:12,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:12,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:14,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:14,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:14,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:14,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:16,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:16,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:16,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:17,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:20,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:23,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:24,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:24,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:24,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:24,300 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:25,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:25,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:28,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:29,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:31,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:31,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:33,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:33,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:34,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:34,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:34,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:34,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:37,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:37,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:39,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:02:39,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:41,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:44,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:44,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:44,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:44,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:44,286 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:45,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:45,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:46,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:46,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:49,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:49,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:53,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:55,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:55,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:55,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:55,360 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:55,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:57,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:57,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:01,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:01,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:02,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:02,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:05,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:03:05,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:03:05,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:03:05,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:03:05,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:03:05,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:07,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:09,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:03:09,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:09,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:03:09,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:03:09,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:03:09,705 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:03:12,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:13,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:03:13,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:17,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:17,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:18,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:23,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:28,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:32,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:32,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:34,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:39,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:03:40,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:45,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:47,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:47,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:50,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:55,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:00,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:02,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:02,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:04,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_a3b1bfcfc8aabde7212f.png with policy now +2023-07-12 04:04:04,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_5c7c719c7e9e5e608018.png with policy now +2023-07-12 04:04:04,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_fdf70bc0fef8dac9c1a0.png with policy now +2023-07-12 04:04:04,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,479 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_67790e930adf85923341.png with policy now +2023-07-12 04:04:04,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:04,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:04,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:04,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:04,758 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/10sycwqx-media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 04:04:04,780 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/68wt8gz6-media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 04:04:04,797 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/18x9tls0-media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 04:04:04,837 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4kul8u3v-media/images/validation_960_67790e930adf85923341.png +2023-07-12 04:04:05,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:05,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 04:04:05,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 04:04:05,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_67790e930adf85923341.png +2023-07-12 04:04:05,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 04:04:05,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:04:05,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:05,760 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5my200m3-media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 04:04:05,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/godsqtxa-media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 04:04:05,813 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zqw4jsfk-media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 04:04:06,018 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c0iy450d-media/images/validation_960_67790e930adf85923341.png +2023-07-12 04:04:09,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:04:09,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:10,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:13,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:14,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:16,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:17,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:17,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:17,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:21,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:21,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:24,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:24,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:24,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:24,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:25,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:25,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:27,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:29,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:32,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:32,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:33,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:33,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:34,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:34,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:34,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:34,600 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:37,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:37,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:39,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:39,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:04:41,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:44,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:44,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:44,996 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:44,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:44,998 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:45,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:45,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:47,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:47,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:49,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:49,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:53,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:55,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:55,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:55,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:55,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:55,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:57,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:57,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:00,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:01,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:02,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:02,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:05,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:05,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:05,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:05,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:05,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:05,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:09,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:05:09,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:09,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:11,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:13,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:16,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:16,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:16,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:16,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:16,529 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:17,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:17,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:17,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:17,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:21,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:21,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:25,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:26,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:26,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:26,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:26,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:26,539 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:29,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:29,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:32,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:33,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:33,228 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:33,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:36,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:36,187 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:36,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:36,189 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:37,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:37,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:38,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:39,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:05:40,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:40,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:40,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:40,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:41,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:41,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:43,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:45,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:48,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:48,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:49,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:54,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:59,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:03,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:03,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:05,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:09,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:06:11,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:16,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:18,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:18,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:21,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:26,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:31,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:33,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:33,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:35,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,917 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_4600d4c65eb0657bf15d.png with policy now +2023-07-12 04:06:35,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_133f183508d881d8f1f8.png with policy now +2023-07-12 04:06:35,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_959d30c344a53f583035.png with policy now +2023-07-12 04:06:35,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_783f685672728f7cab19.png with policy now +2023-07-12 04:06:35,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:06:35,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:06:35,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:06:35,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:06:36,237 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v69gl6yy-media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 04:06:36,254 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2np8c83-media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 04:06:36,264 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5u3dkcly-media/images/validation_970_783f685672728f7cab19.png +2023-07-12 04:06:36,420 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/afmhys1e-media/images/validation_970_959d30c344a53f583035.png +2023-07-12 04:06:36,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:37,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:06:37,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_783f685672728f7cab19.png +2023-07-12 04:06:37,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 04:06:37,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_959d30c344a53f583035.png +2023-07-12 04:06:37,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 04:06:37,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:06:38,405 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v0u6dxju-media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 04:06:38,419 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvzc122e-media/images/validation_970_959d30c344a53f583035.png +2023-07-12 04:06:38,455 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q6meisiw-media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 04:06:38,471 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kj4hwmmo-media/images/validation_970_783f685672728f7cab19.png +2023-07-12 04:06:39,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:06:41,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:42,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:45,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:46,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:06:48,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:48,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:48,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:49,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:53,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:53,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:55,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:06:55,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:06:55,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:06:55,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:06:57,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:06:57,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:58,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:01,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:03,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:03,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:03,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:05,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:06,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:06,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:06,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:06,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:09,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:07:09,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:09,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:09,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:13,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:14,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:15,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:15,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:15,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:15,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:17,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:17,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:18,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:18,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:19,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:21,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:25,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:25,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:25,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:25,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:25,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:26,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:26,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:30,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:30,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:33,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:33,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:34,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:35,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:35,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:35,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:35,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:35,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:38,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:38,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:39,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:07:41,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:42,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:46,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:46,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:46,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:46,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:46,528 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:46,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:48,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:48,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:50,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:50,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:51,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:54,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:56,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:56,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:56,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:56,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:56,745 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:58,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:58,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:02,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:02,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:03,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:03,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:06,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:06,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:08:06,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:08:06,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:08:06,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:08:08,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:09,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:08:10,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:08:10,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:11,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:08:11,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:08:11,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:08:11,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:08:13,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:14,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:08:14,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:18,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:18,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:19,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:24,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:29,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:33,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:33,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:34,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:39,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:08:40,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:45,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:48,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:48,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:50,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:55,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:00,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:03,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:03,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:05,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_5bce29b811f38d81b7a9.png with policy now +2023-07-12 04:09:05,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_4522b255d71fb495f911.png with policy now +2023-07-12 04:09:05,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,594 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_4a474756239679aa6533.png with policy now +2023-07-12 04:09:05,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,595 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_4e23c9690e64cda40444.png with policy now +2023-07-12 04:09:05,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:05,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:05,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:05,621 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:05,925 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/86vki8cg-media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 04:09:05,948 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9anp5i8n-media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 04:09:05,965 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8tdo0wcz-media/images/validation_980_4a474756239679aa6533.png +2023-07-12 04:09:06,080 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mz8pisi8-media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 04:09:06,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:06,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4a474756239679aa6533.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:09:06,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:06,748 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rz230x2s-media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 04:09:06,767 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/30a2fp2t-media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 04:09:06,838 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4uw6wve8-media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 04:09:07,621 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nsai1d5v-media/images/validation_980_4a474756239679aa6533.png +2023-07-12 04:09:09,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:09:12,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:14,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:15,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:17,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:18,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:18,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:18,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:22,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:22,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:24,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:24,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:24,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:24,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:26,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:26,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:28,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:30,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:33,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:34,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:34,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:34,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:34,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:34,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:34,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:34,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:38,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:38,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:39,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:09:39,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:42,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:44,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:44,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:44,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:44,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:44,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:46,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:49,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:49,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:49,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:50,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:54,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:54,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:54,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:54,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:54,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:54,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:55,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:58,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:00,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:02,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:04,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:04,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:04,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:04,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:04,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:04,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:05,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:06,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:06,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:09,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:10:10,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:11,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:14,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:15,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:15,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:15,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:15,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:17,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:18,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:18,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:19,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:19,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:22,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:22,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:25,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:25,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:25,560 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:25,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:26,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:26,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:28,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:30,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:33,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:34,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:35,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:35,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:35,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:35,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:35,982 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:35,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:38,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:38,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:39,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:10:39,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:40,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:40,394 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:40,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:40,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:42,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:42,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:45,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:50,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:50,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:50,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:55,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:00,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:05,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:05,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:06,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:09,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:11:12,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:17,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:20,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:20,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:22,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:27,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:32,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:34,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_e555318c04df0d898c35.png with policy now +2023-07-12 04:11:34,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,902 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_d4658b3a29548774d64f.png with policy now +2023-07-12 04:11:34,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_dd615cbf6d5a7a8a896d.png with policy now +2023-07-12 04:11:34,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_952fc214368e10b389b8.png with policy now +2023-07-12 04:11:34,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:11:34,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:11:34,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:11:34,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:11:35,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:11:35,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 04:11:35,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 04:11:35,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 04:11:35,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 04:11:35,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:11:35,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:35,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:38,077 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mgs1qv8y-media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 04:11:38,142 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gce1fiew-media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 04:11:38,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:39,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:39,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:11:39,257 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lb8d60xt-media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 04:11:39,919 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/trvo9gug-media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 04:11:39,996 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wr443tmh-media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 04:11:41,171 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aauszbr4-media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 04:11:41,552 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y6cvia2o-media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 04:11:41,921 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xww4uylq-media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 04:11:43,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:44,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:45,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:11:49,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:49,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:50,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:50,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:54,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:54,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:11:54,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:54,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:11:54,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:11:54,611 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:11:59,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:11:59,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:59,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:04,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:04,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:04,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:04,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:04,595 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:05,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:05,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:05,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:09,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:12:09,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:09,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:10,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:14,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:14,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:14,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:14,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:14,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:15,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:19,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:19,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:20,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:20,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:21,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:24,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:24,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:24,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:24,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:24,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:26,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:29,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:29,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:31,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:34,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:34,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:34,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:34,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:34,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:35,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:35,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:38,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:39,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:12:39,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:39,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:43,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:44,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:44,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:44,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:44,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:44,717 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:48,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:49,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:49,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:50,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:50,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:54,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:54,279 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:55,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:55,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:55,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:55,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:59,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:59,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:00,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:04,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:05,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:05,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:13:05,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:13:05,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:13:05,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:13:05,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:05,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:09,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:13:09,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:13:09,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:09,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:13:09,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:13:09,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:13:09,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:13:10,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:14,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:13:14,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:15,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:20,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:20,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:20,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:25,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:30,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:35,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:35,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:35,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:39,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:13:41,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:46,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:50,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:50,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:51,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:56,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:01,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:04,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,116 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_1a96794a04b7af41b34e.png with policy now +2023-07-12 04:14:04,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_e0021fa68cabb943f49d.png with policy now +2023-07-12 04:14:04,158 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_87434c7eeda4190bf8d8.png with policy now +2023-07-12 04:14:04,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_bd2b2cab96e55845d6e1.png with policy now +2023-07-12 04:14:04,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:04,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:04,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:04,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:04,452 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j49g4fzt-media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 04:14:04,490 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uhorvb1d-media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 04:14:04,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:04,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 04:14:04,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 04:14:04,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 04:14:04,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 04:14:04,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:14:04,497 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9wn9d9fa-media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 04:14:04,609 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6h89ijj9-media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 04:14:04,845 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ijb7p9dn-media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 04:14:04,906 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i36nkybh-media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 04:14:04,984 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/txr7czh6-media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 04:14:05,060 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2dkoe1rx-media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 04:14:06,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:06,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:07,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:09,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:14:09,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:12,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:13,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:14,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:18,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:19,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:21,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:21,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:23,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:23,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:23,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:23,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:23,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:24,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:24,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:29,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:29,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:34,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:34,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:34,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:34,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:34,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:34,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:35,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:36,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:36,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:39,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:14:39,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:40,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:43,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:43,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:43,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:43,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:44,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:44,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:45,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:49,589 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:50,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:51,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:51,505 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:54,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:54,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:54,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:54,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:54,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:56,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:59,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:59,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:02,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:04,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:04,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:04,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:04,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:04,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:07,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:07,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:07,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:09,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:15:09,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:09,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:13,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:14,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:15,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:15,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:15,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:15,052 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:19,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:19,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:19,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:22,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:22,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:24,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:25,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:25,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:25,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:25,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:25,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:29,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:29,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:30,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:34,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:35,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:35,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:35,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:35,090 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:36,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:37,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:37,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:39,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:15:39,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:39,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:39,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:39,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:39,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:39,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:41,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:44,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:46,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:51,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:52,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:52,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:57,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:02,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:07,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:07,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:07,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:09,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:16:13,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:18,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:22,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:22,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:23,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:28,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:33,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:34,224 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_813871e148b66856b8a5.png with policy now +2023-07-12 04:16:34,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_f52e87f356214ba43ded.png with policy now +2023-07-12 04:16:34,266 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_8213e8a29e4df4a0c573.png with policy now +2023-07-12 04:16:34,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,267 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_1fac8c596692012a9a71.png with policy now +2023-07-12 04:16:34,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:16:34,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:16:34,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:16:34,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:16:34,547 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tqa77qvj-media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 04:16:34,576 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jm7m0jdo-media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 04:16:34,588 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgy46oib-media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 04:16:34,594 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/661cromi-media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 04:16:34,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:16:34,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 04:16:34,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 04:16:34,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 04:16:34,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 04:16:34,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:16:35,376 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1ioaof55-media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 04:16:35,384 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8gbbcprc-media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 04:16:35,400 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2ls9nzwc-media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 04:16:35,624 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ndoiutho-media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 04:16:37,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:37,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:38,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:39,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:16:39,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:43,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:45,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:45,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:16:48,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:50,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:52,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:52,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:54,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:54,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:16:54,511 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:16:54,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:16:54,514 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:16:55,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:16:55,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:59,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:00,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:04,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:04,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:04,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:04,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:04,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:05,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:05,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:07,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:07,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:09,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:17:10,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:10,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:15,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:15,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:15,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:15,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:15,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:16,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:20,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:20,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:21,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:22,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:22,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:25,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:25,070 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:25,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:25,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:25,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:25,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:27,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:30,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:32,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:34,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:34,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:34,876 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:34,878 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:35,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:35,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:37,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:37,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:37,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:39,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:17:40,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:42,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:45,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:45,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:45,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:45,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:45,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:48,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:50,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:50,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:53,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:53,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:53,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:55,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:55,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:55,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:55,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:55,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:59,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:00,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:18:00,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:05,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:05,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:06,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:18:06,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:18:06,132 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:18:06,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:18:08,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:08,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:09,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:18:10,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:18:10,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:10,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:18:10,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:10,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:18:10,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:18:10,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:18:15,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:18:15,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:16,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:21,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:23,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:23,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:27,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:32,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:37,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:38,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:38,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:39,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:18:43,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:48,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:53,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:53,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:53,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:58,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:03,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:05,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_f7760d16dbb61cdadc4b.png with policy now +2023-07-12 04:19:05,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 04:19:05,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 04:19:05,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 04:19:05,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 04:19:05,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:19:05,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_bc519090f4c42a857cf5.png with policy now +2023-07-12 04:19:05,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_d02fa8190f5853b0b648.png with policy now +2023-07-12 04:19:05,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_f59fa5400ab07ea1eeae.png with policy now +2023-07-12 04:19:05,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:05,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:05,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:05,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:05,730 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mc8liz44-media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 04:19:05,771 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gyqkvym0-media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 04:19:05,796 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/14krwzs0-media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 04:19:05,806 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aj3w221o-media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 04:19:06,143 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9zfp7o13-media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 04:19:06,164 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3jz6whro-media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 04:19:06,310 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/le982l75-media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 04:19:06,588 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jw64ai70-media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 04:19:08,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:08,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:08,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:09,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:19:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:14,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:15,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:15,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:19,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:20,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:23,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:24,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:24,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:24,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:24,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:25,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:25,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:25,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:30,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:30,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:33,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:33,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:33,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:33,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:35,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:35,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:36,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:38,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:38,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:39,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:19:40,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:41,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:43,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:43,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:43,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:43,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:45,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:45,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:46,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:50,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:52,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:53,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:53,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:53,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:53,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:53,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:53,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:55,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:55,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:58,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:00,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:03,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:04,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:04,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:04,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:04,004 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:05,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:05,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:08,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:08,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:08,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:09,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:20:10,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:14,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:15,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:15,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:15,143 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:15,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:15,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:15,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:19,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:20,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:23,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:23,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:24,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:25,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:25,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:25,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:25,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:25,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:25,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:30,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:35,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:35,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:35,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:35,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:35,627 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:35,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:35,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:38,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:38,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:39,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:20:40,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:40,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:40,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:40,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:40,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:40,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:40,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:45,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:46,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:51,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:53,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:53,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:56,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:01,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:06,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:09,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:09,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:09,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:21:12,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:17,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:22,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:24,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:24,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:27,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:32,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:35,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_bfd7171975011562507f.png with policy now +2023-07-12 04:21:35,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,053 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_fd568449fa00bd93527b.png with policy now +2023-07-12 04:21:35,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_a95db61e150a9ffc4641.png with policy now +2023-07-12 04:21:35,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_3856c65003790d9a56ab.png with policy now +2023-07-12 04:21:35,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:21:35,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:21:35,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:21:35,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:21:35,377 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nfiglqqz-media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 04:21:35,380 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pb5a5dau-media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 04:21:35,416 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ku0qug25-media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 04:21:35,497 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c0k58urc-media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 04:21:35,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:21:35,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 04:21:35,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 04:21:35,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 04:21:35,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 04:21:35,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:21:36,261 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4j4eyfbg-media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 04:21:36,302 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i2cajqmg-media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 04:21:36,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e9a25pap-media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 04:21:36,353 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/08nsv9z0-media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 04:21:38,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:39,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:21:39,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:39,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:40,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:43,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:45,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:21:45,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:49,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:50,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:54,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:55,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:55,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:55,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:21:55,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:21:55,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:21:55,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:21:55,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:21:55,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:59,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:00,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:05,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:05,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:05,883 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:05,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:05,885 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:05,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:05,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:09,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:22:10,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:10,716 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:10,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:10,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:16,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:16,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:16,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:16,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:16,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:16,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:21,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:21,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:21,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:25,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:25,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:25,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:25,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:25,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:25,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:26,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:26,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:26,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:31,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:32,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:35,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:35,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:35,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:35,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:36,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:36,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:37,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:39,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:22:41,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:41,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:41,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:43,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:46,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:46,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:46,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:46,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:46,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:48,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:51,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:51,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:54,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:56,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:56,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:57,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:57,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:57,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:57,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:59,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:01,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:23:01,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:05,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:06,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:07,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:23:07,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:23:07,429 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:23:07,431 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:23:09,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:23:10,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:11,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:23:11,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:11,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:11,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:23:12,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:23:12,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:23:12,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:23:12,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:23:16,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:16,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:23:16,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:21,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:26,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:26,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:26,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:23:31,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:36,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:39,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:23:41,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:41,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:41,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:23:46,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:51,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:56,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:56,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:56,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:01,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:06,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_15446d27b696fee5751e.png with policy now +2023-07-12 04:24:06,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_9a1046e9395d322b203b.png with policy now +2023-07-12 04:24:06,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_e6fd436066e140c8ca6d.png with policy now +2023-07-12 04:24:06,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_341a0c23b11ea49f2577.png with policy now +2023-07-12 04:24:06,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:06,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:06,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:06,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:07,139 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z056jppi-media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 04:24:07,175 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27tfjrpu-media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 04:24:07,207 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b7wmp72a-media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 04:24:07,216 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/822hl4m2-media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 04:24:07,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:09,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:24:11,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:11,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:11,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 04:24:11,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 04:24:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 04:24:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 04:24:11,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:24:11,765 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xrc6jqm8-media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 04:24:11,766 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/josf8kte-media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 04:24:11,830 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mxslmvcm-media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 04:24:11,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:11,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:11,969 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9z2664lz-media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 04:24:12,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:16,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:16,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:18,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:21,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:24,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:25,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:25,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:25,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:25,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:26,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:26,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:27,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:27,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:29,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:31,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:35,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:35,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:35,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:35,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:35,044 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:36,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:36,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:39,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:24:40,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:41,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:42,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:42,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:44,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:44,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:44,995 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:44,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:45,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:51,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:51,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:55,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:55,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:55,320 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:55,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:56,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:56,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:57,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:57,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:57,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:01,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:02,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:05,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:05,772 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:05,772 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:05,774 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:06,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:06,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:07,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:09,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:25:11,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:12,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:12,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:13,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:16,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:16,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:16,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:16,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:16,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:18,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:21,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:21,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:24,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:26,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:27,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:27,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:28,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:28,070 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:28,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:28,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:30,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:31,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:31,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:35,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:36,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:38,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:38,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:38,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:38,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:39,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:25:41,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:41,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:41,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:42,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:42,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:43,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:43,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:43,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:43,477 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:46,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:46,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:47,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:52,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:57,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:57,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:57,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:02,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:07,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:09,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:26:12,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:12,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:13,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:18,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:23,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:28,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:28,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:29,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:34,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:37,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_55de70da5261f9290e40.png with policy now +2023-07-12 04:26:37,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,622 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_f35c05f1d08c6308ff14.png with policy now +2023-07-12 04:26:37,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_9bfabce8c467677fe54b.png with policy now +2023-07-12 04:26:37,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_1daa5fdb77023a407299.png with policy now +2023-07-12 04:26:37,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:26:37,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:26:37,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:26:37,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:26:37,895 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gxtuxo00-media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 04:26:37,950 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kw17xklj-media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 04:26:37,951 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iqki9t7e-media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 04:26:37,954 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f32n8w7i-media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 04:26:39,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:26:39,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:41,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:26:41,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 04:26:41,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 04:26:41,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 04:26:41,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 04:26:41,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:26:42,225 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wo4kjlhy-media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 04:26:42,246 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sxqh33u9-media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 04:26:42,260 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9uodbpe8-media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 04:26:42,275 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ylp4ehu4-media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 04:26:43,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:43,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:44,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:46,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:26:47,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:26:50,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:51,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:26:55,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:26:57,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:26:57,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:26:57,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:26:57,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:26:58,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:58,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:00,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:01,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:01,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:06,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:06,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:07,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:07,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:07,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:07,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:09,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:27:11,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:11,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:11,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:13,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:13,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:16,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:16,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:18,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:18,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:18,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:18,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:21,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:21,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:21,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:26,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:27,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:28,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:28,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:28,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:28,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:28,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:28,948 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:31,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:31,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:32,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:36,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:38,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:38,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:38,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:38,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:38,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:39,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:27:42,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:42,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:43,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:43,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:43,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:47,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:49,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:49,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:49,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:49,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:49,254 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:52,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:52,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:55,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:57,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:58,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:58,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:59,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:59,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:59,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:59,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:28:00,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:02,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:28:02,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:05,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:07,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:09,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:28:10,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:28:10,109 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:28:10,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:28:10,111 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:28:11,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:12,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:28:12,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:14,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:14,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:28:14,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:28:14,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:28:14,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:28:14,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:28:16,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:17,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:28:17,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:21,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:26,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:29,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:29,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:28:32,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:37,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:39,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:28:43,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:44,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:44,353 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:28:48,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:53,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:58,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:59,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:59,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:04,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:09,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:29:09,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,201 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_197022e2780b05209f12.png with policy now +2023-07-12 04:29:09,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_06ba5b400f585cfe22b9.png with policy now +2023-07-12 04:29:09,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_7be7ad3a7a700e0a150c.png with policy now +2023-07-12 04:29:09,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_23fba533f1fd02e857f8.png with policy now +2023-07-12 04:29:09,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:09,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:09,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:09,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:09,517 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/885zd2w6-media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 04:29:09,537 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0sahtjjz-media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 04:29:09,545 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mq7vmlkt-media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 04:29:09,554 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/exl5eo5i-media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 04:29:10,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:12,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:12,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 04:29:12,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 04:29:12,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 04:29:12,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 04:29:12,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:29:12,723 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dbpc8hbn-media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 04:29:12,747 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hi4lozlj-media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 04:29:12,898 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0nclduzr-media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 04:29:12,908 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hn4ilugs-media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 04:29:14,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:29:14,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:15,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:17,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:19,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:21,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:22,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:26,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:27,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:27,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:27,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:27,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:27,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:29,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:29:29,579 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:32,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:32,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:32,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:37,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:37,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:37,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:37,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:37,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:37,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:37,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:39,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:29:42,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:43,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:45,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:29:45,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:47,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:48,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:48,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:48,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:48,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:49,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:52,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:52,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:54,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:57,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:58,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:58,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:58,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:58,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:00,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:00,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:00,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:02,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:02,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:05,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:07,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:09,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:09,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:09,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:09,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:09,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:30:11,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:12,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:12,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:15,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:15,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:17,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:19,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:19,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:19,286 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:19,288 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:21,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:22,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:22,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:27,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:27,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:28,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:28,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:28,971 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:28,973 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:30,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:30,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:32,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:32,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:33,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:37,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:38,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:38,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:38,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:38,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:38,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:39,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:30:42,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:42,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:43,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:43,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:43,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:43,568 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:43,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:45,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:45,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:47,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:47,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:49,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:54,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:59,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:00,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:00,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:05,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:09,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:31:11,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:15,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:15,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:16,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:21,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:26,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:30,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:30,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:31,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:36,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:37,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:37,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_32f9f6540dbdc7c48f81.png with policy now +2023-07-12 04:31:38,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:38,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_68b196a64c6e776ccd9d.png with policy now +2023-07-12 04:31:38,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:38,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_bd0d7850e520c4cb0100.png with policy now +2023-07-12 04:31:38,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:38,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_a070795b58d5c596307b.png with policy now +2023-07-12 04:31:38,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:31:38,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:31:38,054 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:31:38,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:31:38,271 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ql5afo80-media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 04:31:38,333 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zeijnye-media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 04:31:38,339 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z5q9i5zz-media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 04:31:38,342 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mths02qa-media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 04:31:39,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:31:42,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:42,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:31:42,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:42,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 04:31:42,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 04:31:42,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 04:31:42,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 04:31:42,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:31:43,220 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4eaqe12-media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 04:31:43,241 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ow2maca-media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 04:31:43,259 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m7nllooj-media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 04:31:43,287 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/klq2j92z-media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 04:31:45,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:45,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:47,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:47,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:47,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:31:52,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:53,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:56,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:31:56,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:31:56,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:31:56,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:31:57,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:31:57,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:58,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:01,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:01,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:02,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:04,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:07,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:07,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:07,460 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:07,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:07,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:07,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:09,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:32:09,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:12,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:14,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:16,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:16,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:17,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:17,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:17,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:17,248 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:17,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:17,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:20,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:22,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:25,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:27,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:28,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:28,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:28,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:28,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:30,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:32,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:32,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:32,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:32,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:36,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:37,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:37,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:37,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:37,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:37,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:37,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:39,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:32:41,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:43,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:46,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:47,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:47,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:48,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:48,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:48,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:48,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:48,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:51,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:53,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:53,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:57,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:58,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:59,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:59,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:59,125 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:59,127 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:33:02,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:03,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:33:03,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:03,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:03,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:08,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:08,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:09,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:33:09,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:33:09,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:33:09,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:33:09,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:33:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:33:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:14,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:14,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:33:14,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:33:14,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:33:14,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:33:18,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:33:18,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:18,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:18,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:19,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:24,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:29,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:33,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:33,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:34,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:39,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:33:40,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:45,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:48,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:48,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:50,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:55,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:00,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:03,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:03,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:05,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:09,109 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:34:09,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_1f6acd9fcf45ba3ea19f.png with policy now +2023-07-12 04:34:09,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,230 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_4817a2b823e0b176afd7.png with policy now +2023-07-12 04:34:09,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_66da8376f918e5f31cac.png with policy now +2023-07-12 04:34:09,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_6b2cf11ab85982d34290.png with policy now +2023-07-12 04:34:09,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:09,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:09,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:09,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:09,522 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ouzuahv-media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 04:34:09,571 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mhn9zc25-media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 04:34:09,588 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/40m8p3y0-media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 04:34:09,597 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cv0geh5t-media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 04:34:11,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:13,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:13,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:13,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 04:34:13,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 04:34:13,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 04:34:13,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 04:34:13,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:34:13,698 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zy4s9pss-media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 04:34:13,709 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ppaltm0f-media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 04:34:13,819 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sez5n7i9-media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 04:34:13,864 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fx4cja0g-media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 04:34:16,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:18,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:18,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:18,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:19,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:21,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:23,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:27,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:28,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:28,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:28,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:28,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:28,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:32,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:33,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:33,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:34,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:34,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:37,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:38,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:38,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:38,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:38,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:38,971 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:39,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:34:43,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:43,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:43,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:48,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:48,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:49,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:49,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:49,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:49,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:49,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:49,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:53,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:53,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:53,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:58,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:58,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:59,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:59,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:59,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:59,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:03,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:03,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:04,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:04,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:04,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:08,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:09,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:35:10,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:10,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:10,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:10,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:10,738 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:13,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:13,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:15,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:18,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:19,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:19,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:20,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:20,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:20,207 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:20,209 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:21,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:23,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:23,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:26,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:28,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:30,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:30,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:30,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:30,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:31,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:33,582 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:33,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:35,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:35,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:37,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:38,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:39,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:35:40,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:40,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:40,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:40,626 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:42,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:43,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:43,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:45,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:45,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:45,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:45,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:48,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:48,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:48,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:50,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:50,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:54,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:59,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:04,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:05,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:05,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:09,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:36:09,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:14,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:19,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:20,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:20,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:25,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:30,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:35,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:35,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:35,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:39,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:36:39,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_b8740b576006b9ee3546.png with policy now +2023-07-12 04:36:39,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,361 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_76a3173208a51ee8c312.png with policy now +2023-07-12 04:36:39,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_057775edd61cac3840c0.png with policy now +2023-07-12 04:36:39,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_b585e13abb070740f049.png with policy now +2023-07-12 04:36:39,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:36:39,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:36:39,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:36:39,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:36:39,670 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6b7kof9s-media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 04:36:39,673 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lpakgm1u-media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 04:36:39,676 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e41tyzyk-media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 04:36:39,752 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/scm0l7e1-media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 04:36:40,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:43,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 04:36:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 04:36:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:36:44,134 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o86u4jgp-media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 04:36:44,153 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1s7pqxud-media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 04:36:44,172 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3smzremv-media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 04:36:44,173 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r1umyyrj-media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 04:36:46,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:48,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:49,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:36:50,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:50,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:51,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:53,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:56,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:58,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:59,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:36:59,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:36:59,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:36:59,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:02,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:03,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:03,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:05,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:05,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:08,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:08,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:09,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:37:09,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:09,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:09,541 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:09,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:13,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:13,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:13,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:18,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:18,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:20,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:20,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:20,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:20,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:20,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:20,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:23,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:23,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:24,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:28,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:30,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:30,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:30,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:30,892 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:30,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:33,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:33,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:35,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:35,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:35,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:38,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:39,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:37:40,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:40,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:40,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:40,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:40,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:44,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:44,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:46,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:49,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:50,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:50,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:50,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:50,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:50,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:50,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:51,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:54,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:54,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:57,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:59,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:01,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:38:01,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:38:01,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:38:01,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:38:02,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:04,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:38:04,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:05,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:05,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:08,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:09,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:09,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:38:11,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:38:11,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:38:11,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:38:11,728 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:38:13,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:14,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:38:14,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:16,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:38:16,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:38:16,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:38:16,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:38:19,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:38:19,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:19,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:20,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:20,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:24,112 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:24,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:29,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:34,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:35,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:35,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:39,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:38:40,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:45,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:50,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:50,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:50,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:55,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:00,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:05,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:05,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:05,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:09,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:39:11,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:11,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_b9ea83bc203cb55d812b.png with policy now +2023-07-12 04:39:11,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_ec943a3b531d5c037829.png with policy now +2023-07-12 04:39:11,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_61651029309c609a7521.png with policy now +2023-07-12 04:39:11,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_1ad4a61cbb8909fcb78e.png with policy now +2023-07-12 04:39:11,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:11,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:11,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:11,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:11,596 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l0fg1gpf-media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 04:39:11,638 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p2g16zrg-media/images/validation_1100_61651029309c609a7521.png +2023-07-12 04:39:11,653 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tax1kv77-media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 04:39:11,713 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yisr6r7c-media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 04:39:14,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:14,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 04:39:14,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 04:39:14,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 04:39:14,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_61651029309c609a7521.png +2023-07-12 04:39:14,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:39:14,694 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/470ic08f-media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 04:39:14,703 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qp1w3fcp-media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 04:39:14,707 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6m3i5jxk-media/images/validation_1100_61651029309c609a7521.png +2023-07-12 04:39:14,965 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zbeb97i7-media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 04:39:16,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:19,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:20,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:20,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:21,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:21,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:24,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:26,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:29,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:30,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:30,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:30,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:30,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:31,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:34,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:34,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:35,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:35,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:36,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:39,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:39:39,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:40,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:40,262 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:40,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:40,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:42,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:44,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:44,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:47,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:49,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:50,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:50,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:51,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:51,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:51,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:51,131 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:53,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:54,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:54,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:58,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:59,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:01,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:01,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:01,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:01,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:03,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:04,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:04,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:05,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:05,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:09,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:09,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:40:09,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:12,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:12,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:12,094 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:12,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:14,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:14,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:14,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:19,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:19,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:21,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:21,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:22,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:22,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:22,334 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:22,336 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:24,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:24,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:24,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:29,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:30,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:32,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:32,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:32,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:32,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:34,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:34,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:36,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:36,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:36,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:39,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:40:39,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:41,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:42,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:42,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:42,110 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:42,112 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:44,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:44,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:46,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:46,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:46,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:46,938 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:46,940 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:49,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:49,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:51,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:51,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:51,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:56,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:01,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:06,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:06,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:06,749 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:09,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:41:12,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:17,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:21,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:21,749 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:22,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:27,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:32,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:36,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:36,776 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:37,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:39,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:41:41,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_28d2a849740383813cb6.png with policy now +2023-07-12 04:41:41,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,286 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_b889d46b029a022ecfc9.png with policy now +2023-07-12 04:41:41,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_54b9949ee470b53f0543.png with policy now +2023-07-12 04:41:41,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_eb513f39f1c260344506.png with policy now +2023-07-12 04:41:41,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:41:41,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:41:41,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:41:41,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:41:41,647 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/onq0sv4z-media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 04:41:41,660 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9291c8h0-media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 04:41:41,668 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dzep3ruk-media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 04:41:41,717 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zknqrqt3-media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 04:41:43,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:44,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:41:44,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 04:41:44,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 04:41:44,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 04:41:44,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 04:41:44,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:41:45,178 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9m3xkejl-media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 04:41:45,192 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oov39fny-media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 04:41:45,193 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y0f2gfbf-media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 04:41:45,307 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyfczars-media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 04:41:48,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:49,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:41:51,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:41:51,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:51,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:54,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:54,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:41:59,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:59,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:00,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:00,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:00,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:00,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:04,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:04,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:04,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:07,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:07,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:09,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:42:09,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:09,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:09,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:09,776 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:09,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:09,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:09,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:14,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:14,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:19,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:19,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:19,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:19,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:19,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:19,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:20,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:22,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:22,147 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:24,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:25,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:29,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:30,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:30,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:30,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:30,110 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:31,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:34,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:34,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:36,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:38,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:38,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:39,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:42:39,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:41,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:41,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:41,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:41,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:41,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:45,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:45,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:47,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:50,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:52,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:52,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:52,315 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:52,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:52,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:53,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:53,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:55,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:55,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:57,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:00,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:02,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:43:02,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:02,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:43:02,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:43:02,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:43:05,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:43:05,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:08,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:08,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:08,155 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:09,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:43:10,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:12,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:43:12,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:43:12,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:43:12,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:43:13,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:15,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:43:15,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:17,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:43:17,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:43:17,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:43:17,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:43:19,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:20,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:43:20,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:23,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:23,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:24,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:29,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:34,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:38,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:38,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:39,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:43:39,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:44,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:49,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:53,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:53,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:55,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:00,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:05,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:08,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:08,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:09,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:44:11,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:11,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_3bb3ea14d134cd57deb1.png with policy now +2023-07-12 04:44:11,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,958 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_2c6f306607bc7b9bfd92.png with policy now +2023-07-12 04:44:11,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_8c2839a1d3140c3a7dfe.png with policy now +2023-07-12 04:44:11,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,979 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_fd6b36b89f1e7fd2e78a.png with policy now +2023-07-12 04:44:12,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:12,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:12,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:12,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:12,271 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ic8e69s9-media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 04:44:12,282 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b8m4ctxf-media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 04:44:12,283 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u8pfiyuv-media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 04:44:12,367 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/toi8yswl-media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 04:44:15,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 04:44:15,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:44:15,664 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/knin30hy-media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 04:44:15,678 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4i0zma8-media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 04:44:15,681 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4k84dvv1-media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 04:44:15,701 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0v1899xp-media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 04:44:16,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:20,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:21,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:22,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:23,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:23,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:25,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:28,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:30,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:31,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:31,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:31,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:31,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:33,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:35,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:35,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:38,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:38,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:38,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:39,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:44:40,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:40,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:40,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:40,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:40,937 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:43,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:45,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:45,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:49,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:50,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:50,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:50,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:50,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:50,687 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:53,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:53,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:54,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:55,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:55,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:59,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:00,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:01,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:01,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:01,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:01,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:05,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:05,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:05,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:09,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:09,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:09,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:45:10,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:10,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:12,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:12,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:12,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:12,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:15,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:15,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:16,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:20,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:21,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:23,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:23,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:23,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:23,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:24,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:24,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:25,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:25,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:27,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:30,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:32,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:33,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:33,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:33,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:33,184 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:35,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:35,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:37,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:39,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:45:39,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:39,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:40,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:42,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:42,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:42,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:42,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:42,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:45,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:45,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:48,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:48,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:48,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:48,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:48,071 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:50,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:50,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:53,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:54,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:59,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:04,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:09,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:09,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:46:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:09,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:14,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:19,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:24,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:24,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:25,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:30,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:35,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:39,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:46:39,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:39,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:40,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:42,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_93e81f1f307564b83df8.png with policy now +2023-07-12 04:46:42,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_331aea85c432690f9c0e.png with policy now +2023-07-12 04:46:42,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_44a8006c0888f7a9b714.png with policy now +2023-07-12 04:46:42,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_50f80b0ec1f9f7e17923.png with policy now +2023-07-12 04:46:42,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:46:42,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:46:42,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:46:42,549 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:46:42,823 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b61qzhuz-media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 04:46:42,847 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c1bg5fiv-media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 04:46:42,848 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icsen1n5-media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 04:46:42,881 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w0zhxpwj-media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 04:46:45,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:46:45,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 04:46:45,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 04:46:45,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 04:46:45,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 04:46:45,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:46:46,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:46,194 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x5o8fjqn-media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 04:46:46,201 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3mrskffr-media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 04:46:46,243 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83c6mbmu-media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 04:46:46,255 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d359d2so-media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 04:46:50,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:46:51,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:51,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:46:54,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:54,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:55,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:46:56,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:00,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:01,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:01,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:01,898 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:01,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:01,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:05,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:05,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:07,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:09,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:47:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:09,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:47:10,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:11,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:11,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:11,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:11,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:13,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:15,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:15,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:18,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:20,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:21,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:21,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:21,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:21,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:24,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:25,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:25,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:47:25,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:25,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:29,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:30,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:31,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:31,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:31,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:31,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:35,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:35,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:35,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:39,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:47:40,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:40,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:41,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:41,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:47:42,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:42,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:42,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:42,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:45,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:46,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:46,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:50,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:51,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:52,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:52,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:52,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:52,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:55,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:56,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:56,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:57,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:57,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:01,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:01,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:02,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:48:02,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:48:02,973 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:48:02,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:48:06,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:48:06,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:07,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:09,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:48:11,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:12,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:12,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:12,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:13,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:48:13,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:48:13,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:48:13,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:48:16,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:48:16,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:18,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:18,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:48:18,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:48:18,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:48:18,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:48:21,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:48:21,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:23,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:27,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:27,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:29,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:34,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:39,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:48:40,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:42,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:42,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:45,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:50,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:55,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:57,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:57,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:00,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:05,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:09,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:49:11,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:12,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_1961fdba2d4312886e8c.png with policy now +2023-07-12 04:49:12,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_49f1b6dda25d69bd1eec.png with policy now +2023-07-12 04:49:12,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_a158acb6a30a6a4fcc09.png with policy now +2023-07-12 04:49:12,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_469091e483586cec37b1.png with policy now +2023-07-12 04:49:12,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:12,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:12,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:12,704 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:12,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:12,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:12,943 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cname3sw-media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 04:49:12,969 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pz4i38rj-media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 04:49:12,997 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1va1i1yw-media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 04:49:13,010 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ophx3tba-media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 04:49:16,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:16,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 04:49:16,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 04:49:16,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 04:49:16,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 04:49:16,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:49:16,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:16,778 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x7quqeo5-media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 04:49:16,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qs2drvpr-media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 04:49:16,780 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3njsz0fo-media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 04:49:16,781 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1ly5ch2-media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 04:49:21,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:22,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:22,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:26,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:27,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:27,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:27,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:31,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:32,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:32,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:32,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:32,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:33,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:36,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:36,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:38,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:39,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:49:41,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:41,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:41,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:41,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:41,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:43,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:43,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:44,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:46,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:46,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:49,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:51,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:52,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:52,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:52,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:52,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:54,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:56,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:56,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:59,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:59,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:00,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:01,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:02,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:02,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:02,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:02,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:06,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:06,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:06,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:09,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:50:11,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:12,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:12,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:12,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:12,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:12,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:14,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:50:14,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:16,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:16,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:17,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:21,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:22,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:22,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:22,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:22,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:22,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:26,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:26,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:28,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:29,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:50:29,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:31,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:32,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:32,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:32,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:32,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:33,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:36,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:36,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:39,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:50:39,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:41,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:43,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:43,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:43,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:43,270 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:44,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:45,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:50:45,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:46,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:46,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:48,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:48,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:48,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:48,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:49,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:51,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:51,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:54,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:59,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:00,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:00,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:05,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:09,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:51:11,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:15,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:15,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:16,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:21,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:26,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:30,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:30,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:31,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:36,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:39,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:51:42,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:42,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_9d89ef355de68c0d9ddd.png with policy now +2023-07-12 04:51:42,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_bc32f73444fde5093291.png with policy now +2023-07-12 04:51:42,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_c9a41cef5442cfbcf575.png with policy now +2023-07-12 04:51:42,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_4646158f3cc2ef18d62e.png with policy now +2023-07-12 04:51:42,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:51:42,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:51:42,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:51:42,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:51:43,193 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n9pkzkf5-media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 04:51:43,247 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/szoeplkq-media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 04:51:43,257 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hnczw4cy-media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 04:51:43,266 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjkbsynj-media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 04:51:45,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:45,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:46,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:51:46,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:51:46,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 04:51:46,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 04:51:46,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 04:51:46,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 04:51:46,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:51:47,207 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q0pl6f7d-media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 04:51:47,229 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cnpn75v2-media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 04:51:47,242 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/khc0dobf-media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 04:51:47,276 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2o9zlke7-media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 04:51:47,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:51,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:51:52,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:51:53,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:56,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:51:58,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:00,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:00,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:01,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:02,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:02,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:02,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:02,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:04,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:06,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:06,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:09,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:52:09,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:11,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:11,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:11,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:11,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:11,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:11,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:14,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:15,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:15,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:16,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:20,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:21,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:22,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:22,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:22,516 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:22,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:26,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:26,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:26,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:30,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:30,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:31,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:31,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:33,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:33,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:33,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:33,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:36,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:36,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:37,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:39,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:52:41,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:42,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:43,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:43,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:43,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:43,291 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:45,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:45,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:47,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:47,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:47,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:52,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:53,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:53,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:53,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:53,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:53,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:57,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:57,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:58,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:00,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:00,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:02,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:03,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:53:03,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:03,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:53:03,551 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:53:03,553 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:53:07,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:53:07,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:09,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:53:09,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:12,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:13,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:53:13,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:53:13,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:53:13,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:53:14,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:15,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:15,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:17,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:53:17,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:18,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:53:18,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:53:18,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:53:18,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:53:19,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:22,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:53:22,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:24,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:29,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:30,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:30,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:35,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:39,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:53:41,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:45,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:45,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:46,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:51,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:00,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:00,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:02,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:07,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:09,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:54:12,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_a90fd9580e1c1e24f448.png with policy now +2023-07-12 04:54:12,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_6b84704eb15e35e5f9a9.png with policy now +2023-07-12 04:54:12,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_6e5ebba16091d67e0c49.png with policy now +2023-07-12 04:54:12,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_9a7e5eb802701258688c.png with policy now +2023-07-12 04:54:12,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:12,849 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:12,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:12,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:12,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:13,149 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dbdnof94-media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 04:54:13,161 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3dkexbud-media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 04:54:13,168 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0u22r6yp-media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 04:54:13,191 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6q84o7pz-media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 04:54:15,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:15,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:17,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 04:54:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 04:54:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 04:54:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 04:54:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:54:17,838 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jajy3kz4-media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 04:54:17,840 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ybekej2n-media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 04:54:17,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:17,874 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p9o7tl56-media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 04:54:17,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e81ohyl3-media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 04:54:22,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:23,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:23,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:27,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:28,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:30,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:30,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:31,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:31,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:31,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:31,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:32,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:32,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:33,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:37,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:39,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:54:39,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:41,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:41,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:41,556 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:41,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:42,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:42,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:44,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:45,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:45,923 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:47,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:50,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:52,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:52,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:52,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:52,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:52,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:55,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:57,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:00,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:01,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:01,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:02,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:02,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:02,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:02,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:02,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:02,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:05,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:07,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:09,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:55:11,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:12,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:12,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:12,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:12,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:12,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:16,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:16,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:16,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:17,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:17,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:22,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:22,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:23,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:23,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:23,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:23,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:27,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:27,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:27,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:31,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:31,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:32,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:33,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:37,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:37,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:55:40,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:40,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:40,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:40,868 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:40,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:55:42,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:42,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:44,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:44,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:44,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:44,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:46,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:46,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:55:46,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:46,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:48,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:48,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:48,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:48,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:51,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:52,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:52,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:56,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:01,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:01,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:01,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:01,789 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:06,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:09,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:56:12,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:16,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:16,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:16,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:17,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:22,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:27,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:31,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:31,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:31,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:33,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:38,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:39,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:56:43,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_a451b10b1cddeff35521.png with policy now +2023-07-12 04:56:43,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_2bc25132345cfabe28d9.png with policy now +2023-07-12 04:56:43,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_f9f3ae84aa2fd03f0c14.png with policy now +2023-07-12 04:56:43,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,353 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_e76ace79b32cf06e3105.png with policy now +2023-07-12 04:56:43,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:56:43,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:56:43,395 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:56:43,397 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:56:43,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j4d6bmxw-media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 04:56:43,655 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pp5i3vsz-media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 04:56:43,691 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i09ncfdm-media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 04:56:43,727 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5oetk4ej-media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 04:56:44,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:46,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:46,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:46,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:47,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:56:47,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:56:47,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 04:56:47,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 04:56:47,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 04:56:47,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 04:56:47,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:56:48,251 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eh6pgpa7-media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 04:56:48,253 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/765hz2iu-media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 04:56:48,276 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i3xiuagw-media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 04:56:48,279 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iu8dp028-media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 04:56:49,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:52,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:56:53,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:56:55,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:57,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:00,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:01,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:01,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:01,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:02,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:03,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:03,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:03,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:03,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:05,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:07,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:07,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:09,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:57:10,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:12,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:13,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:13,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:13,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:13,111 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:16,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:17,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:17,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:17,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:17,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:17,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:22,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:22,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:22,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:22,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:22,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:22,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:22,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:27,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:28,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:32,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:32,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:32,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:32,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:32,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:32,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:32,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:32,949 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:33,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:33,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:38,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:38,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:39,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:57:43,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:43,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:43,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:43,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:43,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:43,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:47,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:47,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:47,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:48,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:48,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:49,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:53,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:53,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:53,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:53,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:53,347 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:54,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:58,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:58,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:59,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:02,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:02,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:02,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:03,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:03,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:58:03,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:58:03,851 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:58:03,853 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:58:04,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:08,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:58:08,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:09,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:58:10,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:13,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:14,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:58:14,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:58:14,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:58:14,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:58:15,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:17,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:17,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:17,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:18,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:58:18,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:19,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:58:19,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:58:19,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:58:19,184 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:58:20,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:23,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:58:23,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:25,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:30,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:32,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:32,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:32,761 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:35,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:39,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:58:41,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:46,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:47,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:47,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:47,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:52,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:57,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:02,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:02,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:02,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:02,927 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:07,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:09,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:59:13,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:13,404 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,404 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_d085f51cf12dae1345cd.png with policy now +2023-07-12 04:59:13,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_f8627735c220e9667e1c.png with policy now +2023-07-12 04:59:13,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_20948b93a6c00143c135.png with policy now +2023-07-12 04:59:13,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 04:59:13,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 04:59:13,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:59:13,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_7b2ab005b9837744864c.png with policy now +2023-07-12 04:59:13,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:13,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:13,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:13,503 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:13,742 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5vqejp2n-media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 04:59:13,744 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fkepbf8b-media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 04:59:13,745 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpm21z2y-media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 04:59:13,805 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6acm935-media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 04:59:14,149 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gip21clf-media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 04:59:14,167 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wqop05je-media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 04:59:17,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:17,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:17,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:18,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:59:18,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:18,825 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83zuq9ip-media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 04:59:18,832 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/holu3ce4-media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 04:59:23,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:23,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:24,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:28,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:29,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:31,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:31,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:31,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:31,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:33,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:33,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:33,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:33,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:33,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:35,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:38,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:39,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:59:40,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:41,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:41,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:41,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:41,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:43,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:43,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:45,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:48,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:49,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:49,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:49,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:50,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:51,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:51,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:51,971 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:51,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:53,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:53,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:56,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:58,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:01,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:02,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:02,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:02,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:02,893 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:03,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:03,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:05,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:05,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:05,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:06,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:08,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:09,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:00:11,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:13,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:13,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:13,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:13,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:13,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:13,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:17,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:18,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:20,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:20,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:20,159 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:22,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:22,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:22,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:22,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:22,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:23,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:23,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:28,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:28,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:33,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:33,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:33,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:33,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:33,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:33,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:33,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:35,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:35,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:35,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:38,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:38,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:39,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:00:43,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:44,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:44,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:44,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:44,645 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:44,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:48,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:48,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:49,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:49,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:49,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:49,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:49,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:50,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:50,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:50,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:53,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:53,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:54,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:59,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:04,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:05,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:05,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:05,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:09,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:01:10,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:15,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:20,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:20,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:20,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:20,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:25,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:30,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:35,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:35,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:35,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:35,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:39,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:01:41,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:43,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_f7abacf0c65108ebd291.png with policy now +2023-07-12 05:01:43,451 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_c631ebe8975b90c43163.png with policy now +2023-07-12 05:01:43,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_138faee5a37f54662918.png with policy now +2023-07-12 05:01:43,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_597e7e8766e45fd365d0.png with policy now +2023-07-12 05:01:43,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:01:43,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:01:43,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:01:43,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:01:43,803 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpsn8kt2-media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 05:01:43,839 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/38glgnu7-media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 05:01:43,852 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grojtuti-media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 05:01:43,856 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nv49fg2e-media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 05:01:43,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:01:43,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 05:01:43,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 05:01:43,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 05:01:43,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 05:01:43,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:01:44,346 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eytsjzyp-media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 05:01:44,361 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pzxec5l9-media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 05:01:44,380 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l52pfi4j-media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 05:01:44,385 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qbdjm7bw-media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 05:01:46,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:48,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:01:50,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:50,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:50,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:52,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:53,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:01:54,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:01:57,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:59,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:02,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:02,220 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:02,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:02,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:03,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:04,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:04,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:06,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:06,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:06,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:08,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:09,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:09,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:02:12,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:12,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:12,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:12,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:14,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:14,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:14,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:19,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:19,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:21,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:21,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:21,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:22,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:22,263 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:22,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:22,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:24,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:24,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:24,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:29,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:30,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:32,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:32,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:32,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:32,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:34,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:34,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:35,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:36,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:36,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:36,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:39,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:39,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:02:40,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:42,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:42,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:42,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:42,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:44,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:44,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:46,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:49,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:51,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:52,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:52,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:52,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:52,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:52,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:52,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:52,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:54,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:54,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:56,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:59,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:01,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:03,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:03:03,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:03:03,455 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:03:03,457 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:03:04,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:03:04,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:07,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:07,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:07,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:07,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:09,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:10,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:03:13,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:13,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:03:13,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:03:13,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:03:13,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:03:14,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:03:14,279 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:18,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:18,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:03:18,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:03:18,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:03:18,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:03:19,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:03:19,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:22,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:22,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:23,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:24,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:28,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:33,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:37,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:37,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:37,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:39,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:03:39,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:44,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:49,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:52,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:52,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:52,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:55,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:00,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:05,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:07,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:07,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:07,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:09,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:04:10,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:12,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_707f52b23811c66eb81f.png with policy now +2023-07-12 05:04:12,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,820 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_095c7719d5d88d3a784e.png with policy now +2023-07-12 05:04:12,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_509f77863e0d740b58c5.png with policy now +2023-07-12 05:04:12,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_c158e5c7fd160b30b673.png with policy now +2023-07-12 05:04:12,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:12,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:12,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:12,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:13,136 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aw2kp4l1-media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 05:04:13,138 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1jtazdkv-media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 05:04:13,146 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ggaxzog-media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 05:04:13,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mj3r3iiv-media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 05:04:14,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:14,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 05:04:14,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 05:04:14,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 05:04:14,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 05:04:14,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:04:14,861 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q33bgrej-media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 05:04:14,862 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rv4oeqtr-media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 05:04:14,931 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m5bchzqz-media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 05:04:14,942 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y52vfqh1-media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 05:04:15,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:19,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:21,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:22,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:22,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:22,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:22,961 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:24,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:26,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:29,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:31,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:31,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:31,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:31,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:31,920 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:34,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:34,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:37,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:38,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:38,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:38,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:39,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:04:39,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:41,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:41,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:41,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:41,916 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:42,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:44,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:44,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:48,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:49,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:52,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:52,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:52,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:52,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:53,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:54,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:54,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:54,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:54,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:54,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:58,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:59,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:02,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:02,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:02,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:02,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:04,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:04,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:04,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:09,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:05:09,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:10,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:10,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:10,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:10,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:05:13,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:13,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:13,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:13,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:14,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:14,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:15,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:19,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:21,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:23,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:23,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:23,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:23,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:24,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:24,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:25,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:25,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:25,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:05:26,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:29,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:32,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:33,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:33,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:33,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:33,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:34,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:34,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:37,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:39,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:05:39,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:41,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:41,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:41,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:05:42,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:44,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:44,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:44,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:44,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:44,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:44,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:48,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:48,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:48,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:48,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:48,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:49,831 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:49,831 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:54,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:54,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:56,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:56,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:56,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:00,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:05,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:09,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:06:10,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:11,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:11,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:11,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:16,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:21,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:26,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:26,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:26,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:27,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:32,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:37,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:39,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:06:41,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:41,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:41,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:42,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:43,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,316 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_1ba924a90889d370d938.png with policy now +2023-07-12 05:06:43,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,330 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_25ef2dfcdcaf9ff09dea.png with policy now +2023-07-12 05:06:43,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,350 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_a311e64f75624628145d.png with policy now +2023-07-12 05:06:43,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_75f318a5faeb28d48e50.png with policy now +2023-07-12 05:06:43,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:06:43,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:06:43,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:06:43,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:06:43,652 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1vgjckud-media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 05:06:43,670 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e0ofbfor-media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 05:06:43,750 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yvmyw2tk-media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 05:06:43,806 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hwp4syfg-media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 05:06:45,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:06:45,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 05:06:45,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 05:06:45,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 05:06:45,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 05:06:45,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:06:45,426 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nqxa75ww-media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 05:06:45,445 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/plwgdw74-media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 05:06:45,462 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u5504xaj-media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 05:06:45,474 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xqvv7wef-media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 05:06:48,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:50,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:06:53,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:06:53,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:55,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:06:56,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:56,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:56,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:58,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:00,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:02,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:02,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:02,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:02,352 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:04,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:05,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:05,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:09,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:07:10,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:10,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:11,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:11,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:11,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:12,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:12,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:12,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:12,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:15,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:15,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:15,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:20,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:21,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:22,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:22,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:22,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:22,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:25,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:25,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:26,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:27,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:27,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:27,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:30,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:31,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:33,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:33,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:33,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:33,694 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:35,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:35,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:37,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:39,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:07:40,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:42,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:42,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:42,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:42,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:44,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:44,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:44,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:44,153 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:45,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:45,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:47,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:50,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:53,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:53,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:53,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:53,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:53,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:55,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:55,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:58,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:58,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:58,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:58,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:00,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:03,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:08:03,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:08:03,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:08:03,619 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:08:04,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:05,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:08:05,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:09,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:08:10,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:13,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:13,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:13,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:08:13,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:08:13,942 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:08:13,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:08:13,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:08:15,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:08:15,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:15,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:18,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:08:18,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:08:18,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:08:18,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:08:20,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:08:20,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:21,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:26,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:28,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:28,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:28,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:08:31,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:36,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:39,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:08:42,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:43,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:43,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:43,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:08:47,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:52,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:57,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:58,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:58,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:58,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:02,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:07,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:09,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:09:13,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:13,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_1015b192327aea438f8a.png with policy now +2023-07-12 05:09:13,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_b37517440f90e43b7657.png with policy now +2023-07-12 05:09:13,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_65b3c4bdf334afcfec36.png with policy now +2023-07-12 05:09:13,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_624173894a8202fd049b.png with policy now +2023-07-12 05:09:13,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:13,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:13,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:13,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:13,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:09:13,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:09:13,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:13,685 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/98ze89pp-media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 05:09:13,696 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mtgomn5z-media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 05:09:13,758 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gckxd9fh-media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 05:09:13,760 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t0wwx2ap-media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 05:09:15,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:15,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 05:09:15,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 05:09:15,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 05:09:15,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 05:09:15,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:09:15,962 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pycqbyel-media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 05:09:15,967 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5pyta22m-media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 05:09:15,977 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vtzjj1go-media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 05:09:16,046 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vpko4ipa-media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 05:09:18,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:20,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:23,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:24,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:25,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:28,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:09:28,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:09:28,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:29,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:30,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:32,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:32,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:32,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:32,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:34,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:35,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:35,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:39,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:09:40,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:40,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:42,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:42,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:42,746 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:42,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:44,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:09:44,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:09:44,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:45,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:45,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:45,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:50,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:50,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:53,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:53,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:53,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:53,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:55,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:55,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:55,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:00,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:00,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:00,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:00,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:00,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:03,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:03,313 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:03,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:03,315 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:05,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:05,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:05,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:09,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:10:10,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:11,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:13,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:13,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:13,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:13,344 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:15,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:15,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:15,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:15,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:15,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:16,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:20,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:22,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:24,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:24,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:24,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:24,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:28,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:30,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:30,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:30,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:30,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:33,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:35,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:35,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:35,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:35,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:35,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:35,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:38,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:39,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:10:40,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:44,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:45,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:45,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:45,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:45,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:45,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:45,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:45,594 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:45,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:45,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:50,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:50,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:50,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:50,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:50,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:50,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:50,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:55,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:56,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:00,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:00,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:00,444 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:01,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:06,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:09,187 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:11:12,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:15,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:15,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:15,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:17,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:22,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:27,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:30,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:30,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:30,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:36,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:41,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:44,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:11:44,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:45,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,074 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_bed029a53fc783aa1494.png with policy now +2023-07-12 05:11:45,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,084 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_82f6db57a6a62b37c416.png with policy now +2023-07-12 05:11:45,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,094 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_0a392241e2b0e5c56de0.png with policy now +2023-07-12 05:11:45,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,104 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_fe7e092e619d5a60940f.png with policy now +2023-07-12 05:11:45,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:11:45,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:11:45,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:11:45,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:11:45,637 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kjyjvhd5-media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 05:11:45,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:45,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:45,796 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8rtq4xck-media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 05:11:45,848 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bcv730lg-media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 05:11:46,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:11:46,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 05:11:46,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 05:11:46,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 05:11:46,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 05:11:46,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:11:46,136 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/epdbk4qe-media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 05:11:46,669 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3ezqrcc-media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 05:11:46,683 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4k0digv4-media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 05:11:47,049 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eef0wf0q-media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 05:11:47,107 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zwnvswl-media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 05:11:49,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:51,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:11:54,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:11:55,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:56,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:01,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:01,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:01,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:01,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:01,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:04,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:04,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:04,248 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:04,250 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:06,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:06,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:07,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:09,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:12:11,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:13,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:13,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:13,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:13,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:13,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:16,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:16,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:16,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:16,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:16,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:18,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:21,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:23,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:24,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:24,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:24,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:24,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:26,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:26,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:29,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:31,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:31,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:31,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:31,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:34,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:34,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:34,640 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:34,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:34,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:36,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:36,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:39,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:12:40,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:41,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:44,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:44,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:44,449 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:44,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:45,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:46,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:46,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:47,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:47,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:47,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:50,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:51,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:55,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:55,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:55,249 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:55,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:56,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:56,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:56,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:01,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:02,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:02,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:02,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:02,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:05,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:13:05,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:13:05,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:13:05,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:13:06,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:13:06,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:07,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:09,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:13:11,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:13,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:16,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:16,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:13:16,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:13:16,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:13:16,465 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:13:17,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:17,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:17,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:18,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:21,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:13:21,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:13:21,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:13:21,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:13:21,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:13:21,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:23,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:26,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:28,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:32,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:32,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:32,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:34,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:39,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:13:40,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:45,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:47,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:47,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:47,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:50,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:55,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:00,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:02,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:14:02,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:02,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:05,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:09,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:14:11,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:15,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_9038ef38623aaf3b1297.png with policy now +2023-07-12 05:14:15,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_4fdefa2382f1383aa03b.png with policy now +2023-07-12 05:14:15,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_9b87b6c7ef2509f2091e.png with policy now +2023-07-12 05:14:15,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_563721340a1b7779ebb1.png with policy now +2023-07-12 05:14:15,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:15,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:15,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:15,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:15,726 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k3rm3soh-media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 05:14:15,773 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hy24zmfj-media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 05:14:15,778 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6hysak8e-media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 05:14:15,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/inyzzxfo-media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 05:14:16,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:16,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:16,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 05:14:16,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 05:14:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 05:14:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 05:14:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:14:16,990 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/irks8uit-media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 05:14:17,014 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0f0rk2j4-media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 05:14:17,026 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t02k5qcj-media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 05:14:17,030 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u1urkopz-media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 05:14:17,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:14:17,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:17,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:21,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:22,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:25,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:27,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:27,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:32,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:33,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:14:33,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:33,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:33,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:35,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:35,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:35,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:35,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:38,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:39,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:14:39,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:39,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:43,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:45,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:45,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:45,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:45,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:45,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:45,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:48,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:48,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:49,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:51,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:54,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:56,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:56,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:56,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:56,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:57,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:57,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:00,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:03,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:03,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:03,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:03,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:05,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:06,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:06,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:06,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:06,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:09,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:15:09,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:09,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:11,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:15,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:16,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:16,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:16,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:16,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:16,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:18,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:18,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:18,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:22,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:22,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:22,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:26,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:26,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:26,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:26,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:27,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:28,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:28,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:33,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:33,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:33,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:33,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:34,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:36,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:36,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:36,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:36,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:38,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:39,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:15:40,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:40,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:44,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:46,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:47,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:47,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:47,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:47,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:48,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:48,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:48,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:49,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:52,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:52,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:52,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:52,219 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:52,219 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:52,221 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:58,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:58,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:59,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:03,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:03,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:03,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:05,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:09,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:16:11,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:16,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:18,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:18,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:18,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:21,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:26,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:31,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:33,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:33,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:36,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:39,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:16:42,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:46,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_a92796d341fe8362362e.png with policy now +2023-07-12 05:16:46,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_7e96d8ec566f76718aea.png with policy now +2023-07-12 05:16:46,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_0ffe8cb158ff46492db0.png with policy now +2023-07-12 05:16:46,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_4abee66cab5fc4486094.png with policy now +2023-07-12 05:16:46,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:16:46,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:16:46,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:16:46,861 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:16:47,087 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tjqy2dxd-media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 05:16:47,117 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0dayqkx1-media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 05:16:47,117 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9zf987a3-media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 05:16:47,119 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83mc26vf-media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 05:16:47,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:48,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:52,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:16:52,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:16:52,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 05:16:52,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 05:16:52,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 05:16:52,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 05:16:52,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:16:52,760 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mju1skgl-media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 05:16:52,765 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uec25tgp-media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 05:16:52,777 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q3ub8ws7-media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 05:16:52,806 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6pa0jpxe-media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 05:16:53,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:56,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:16:58,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:16:59,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:03,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:03,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:03,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:04,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:04,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:05,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:05,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:05,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:05,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:09,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:17:10,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:10,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:10,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:15,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:16,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:16,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:16,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:16,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:16,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:16,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:18,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:18,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:18,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:21,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:22,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:26,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:26,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:26,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:26,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:26,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:28,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:28,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:31,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:33,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:33,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:33,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:34,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:37,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:37,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:37,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:37,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:37,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:39,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:17:40,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:40,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:42,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:46,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:47,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:47,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:47,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:47,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:47,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:48,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:48,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:48,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:52,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:52,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:52,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:57,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:57,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:57,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:57,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:58,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:58,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:58,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:03,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:03,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:03,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:03,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:04,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:07,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:18:07,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:18:07,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:18:07,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:18:08,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:09,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:18:10,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:18:10,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:13,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:16,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:17,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:18:17,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:18:17,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:18:17,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:18:18,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:18,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:18,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:19,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:22,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:18:22,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:18:22,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:18:22,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:18:22,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:18:22,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:25,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:28,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:30,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:33,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:33,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:33,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:35,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:39,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:18:41,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:46,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:48,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:48,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:48,629 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:51,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:01,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:03,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:03,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:03,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:07,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:09,204 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:19:13,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:16,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_e893310e8c740aedacd9.png with policy now +2023-07-12 05:19:16,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_3930a2a71b1d2e79ebf6.png with policy now +2023-07-12 05:19:16,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_709f8444e3a90e20584a.png with policy now +2023-07-12 05:19:16,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_52dd5c4bac63f8392a24.png with policy now +2023-07-12 05:19:16,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:16,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:16,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:16,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:16,737 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/guhsmjtl-media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 05:19:16,778 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eo1jbvxp-media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 05:19:16,789 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bcsd5g3b-media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 05:19:16,811 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fs1amvrv-media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 05:19:16,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:16,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 05:19:16,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 05:19:16,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 05:19:16,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 05:19:16,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:19:17,247 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yjmo5jrs-media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 05:19:17,255 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9iszdyi-media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 05:19:17,275 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/svetvfjb-media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 05:19:17,306 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3s5r03yp-media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 05:19:18,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:19,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:19,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:19,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:22,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:24,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:26,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:28,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:29,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:34,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:34,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:34,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:34,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:34,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:36,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:36,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:36,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:36,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:39,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:19:39,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:40,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:40,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:44,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:46,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:46,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:46,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:46,393 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:46,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:46,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:49,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:49,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:49,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:50,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:52,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:55,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:56,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:56,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:56,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:56,191 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:58,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:58,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:00,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:04,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:04,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:04,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:04,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:06,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:06,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:06,335 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:06,335 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:06,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:09,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:20:10,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:10,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:11,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:16,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:16,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:16,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:16,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:16,880 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:17,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:17,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:19,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:19,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:19,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:21,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:23,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:26,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:27,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:27,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:27,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:27,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:29,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:29,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:32,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:34,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:34,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:34,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:35,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:37,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:37,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:37,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:37,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:37,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:39,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:20:41,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:41,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:42,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:47,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:47,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:47,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:47,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:47,676 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:47,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:49,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:49,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:49,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:52,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:52,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:52,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:52,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:53,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:53,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:53,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:58,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:59,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:21:03,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:04,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:04,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:04,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:09,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:09,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:21:14,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:19,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:19,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:19,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:19,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:24,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:29,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:34,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:35,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:35,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:35,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:39,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:21:40,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:45,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:47,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_39451786e3d6986a15e5.png with policy now +2023-07-12 05:21:47,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_7507600afa47a7721bc0.png with policy now +2023-07-12 05:21:47,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_239d53a80d648e3b5d74.png with policy now +2023-07-12 05:21:47,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_007f79d4a566d7c33ea0.png with policy now +2023-07-12 05:21:47,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:21:47,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:21:47,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:21:47,212 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:21:47,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:21:47,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 05:21:47,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 05:21:47,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 05:21:47,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 05:21:47,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:21:47,449 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/73xoa915-media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 05:21:47,458 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/185yktdd-media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 05:21:47,463 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g7syc8mb-media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 05:21:47,526 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p9pw1med-media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 05:21:47,870 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zw021il2-media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 05:21:47,894 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vybt3p8r-media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 05:21:48,206 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4vvobbx8-media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 05:21:48,271 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/us08xq4x-media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 05:21:50,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:50,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:50,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:50,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:53,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:21:55,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:57,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:21:59,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:01,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:05,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:05,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:05,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:05,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:06,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:06,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:06,402 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:06,404 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:07,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:09,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:22:11,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:11,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:12,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:15,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:15,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:15,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:15,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:17,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:17,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:18,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:20,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:20,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:20,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:23,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:24,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:25,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:25,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:25,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:25,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:29,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:29,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:29,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:35,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:35,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:35,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:35,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:35,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:35,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:35,525 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:35,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:39,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:22:40,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:41,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:41,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:45,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:45,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:45,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:45,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:45,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:47,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:47,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:50,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:50,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:50,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:50,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:53,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:55,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:55,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:55,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:55,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:56,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:59,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:59,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:02,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:05,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:23:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:05,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:06,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:23:06,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:23:06,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:23:06,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:23:07,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:09,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:23:11,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:23:11,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:12,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:16,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:23:16,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:23:16,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:23:16,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:23:17,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:23:17,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:17,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:20,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:20,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:21,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:23:21,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:23:21,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:23:21,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:23:23,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:23,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:23:23,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:28,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:29,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:33,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:35,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:23:35,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:35,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:38,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:39,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:23:44,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:49,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:50,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:50,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:23:50,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:54,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:59,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:04,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:24:05,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:09,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:24:10,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:15,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:16,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_91a32c15ebfc912c2201.png with policy now +2023-07-12 05:24:16,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_76c909633b7c82a1c57d.png with policy now +2023-07-12 05:24:16,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_d09f421be121a991d346.png with policy now +2023-07-12 05:24:16,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_f520f48ddc3348a6c6bd.png with policy now +2023-07-12 05:24:16,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:16,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:16,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:16,206 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:16,434 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fqhebolb-media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 05:24:16,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uucsgcb3-media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 05:24:16,469 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/utxhs1iq-media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 05:24:16,480 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1mdk3wfy-media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 05:24:17,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:17,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 05:24:17,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 05:24:17,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 05:24:17,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 05:24:17,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:24:18,247 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gc8za9ny-media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 05:24:18,257 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rlfhdqh6-media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 05:24:18,266 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7fpujty9-media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 05:24:18,274 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x7fg14xo-media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 05:24:20,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:20,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:20,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:23,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:25,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:26,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:29,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:31,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:35,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:35,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:35,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:35,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:35,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:35,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:35,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:35,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:36,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:39,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:24:41,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:42,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:44,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:44,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:44,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:44,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:47,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:47,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:47,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:51,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:24:51,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:51,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:53,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:53,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:54,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:54,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:54,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:54,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:58,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:59,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:59,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:03,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:04,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:04,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:04,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:04,340 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:05,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:05,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:06,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:06,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:06,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:09,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:09,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:25:11,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:14,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:14,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:14,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:14,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:14,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:18,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:18,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:19,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:21,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:21,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:21,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:24,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:25,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:25,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:25,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:25,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:25,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:30,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:30,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:30,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:36,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:36,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:36,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:36,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:36,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:36,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:36,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:36,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:36,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:39,219 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:25:42,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:42,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:42,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:46,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:46,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:46,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:46,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:47,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:48,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:48,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:51,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:51,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:51,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:51,083 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:51,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:51,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:51,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:53,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:54,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:54,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:58,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:03,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:07,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:26:07,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:07,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:08,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:09,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:26:13,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:18,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:22,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:26:22,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:22,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:24,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:29,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:34,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:37,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:37,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:26:37,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:39,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:26:39,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:44,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:45,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_c67265e3a1dfdfb99387.png with policy now +2023-07-12 05:26:45,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_e8b496a8b73ca169dce8.png with policy now +2023-07-12 05:26:45,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_ad7f6abdff5cb1890622.png with policy now +2023-07-12 05:26:45,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_499d8794ae851f2ca7c3.png with policy now +2023-07-12 05:26:45,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:26:45,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:26:45,732 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:26:45,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:26:46,036 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xyiubq3i-media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 05:26:46,047 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pojd9kfo-media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 05:26:46,054 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dt801efh-media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 05:26:46,059 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kur41fri-media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 05:26:48,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:26:48,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 05:26:48,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 05:26:48,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 05:26:48,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 05:26:48,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:26:48,729 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i61xmc7f-media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 05:26:48,734 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sq4bx3wb-media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 05:26:48,767 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/updq52uf-media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 05:26:48,777 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mtr3i4sb-media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 05:26:49,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:52,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:52,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:54,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:26:54,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:54,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:00,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:00,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:03,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:03,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:03,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:03,950 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:05,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:06,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:06,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:07,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:07,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:07,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:09,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:27:10,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:12,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:13,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:13,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:13,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:13,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:16,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:18,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:18,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:22,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:23,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:23,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:23,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:23,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:23,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:23,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:24,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:24,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:27,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:30,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:32,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:34,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:34,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:34,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:34,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:36,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:36,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:38,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:38,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:38,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:38,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:39,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:27:42,503 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:43,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:45,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:45,262 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:45,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:45,264 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:48,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:48,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:49,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:53,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:53,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:53,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:54,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:55,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:55,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:55,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:55,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:55,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:00,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:00,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:00,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:06,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:28:06,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:06,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:28:06,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:28:06,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:06,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:06,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:08,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:08,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:08,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:09,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:28:11,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:12,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:16,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:28:16,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:28:16,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:28:16,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:17,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:18,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:18,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:20,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:28:20,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:28:20,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:28:20,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:22,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:23,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:24,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:24,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:28,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:33,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:38,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:38,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:38,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:38,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:39,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:28:44,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:49,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:53,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:53,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:53,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:54,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:59,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:04,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:08,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:08,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:08,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:09,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:29:10,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:15,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:15,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,381 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_6d25eb3589d810ed3440.png with policy now +2023-07-12 05:29:15,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_295181c2c8f74bdb7122.png with policy now +2023-07-12 05:29:15,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_d49ed1edb101f8e16c80.png with policy now +2023-07-12 05:29:15,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_47d0444bbe383a2f3ed2.png with policy now +2023-07-12 05:29:15,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:15,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:15,455 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:15,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:15,692 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mgp1sv3d-media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 05:29:15,697 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpwjknuj-media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 05:29:15,715 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/elbxqpu0-media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 05:29:15,727 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5svabk9h-media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 05:29:18,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:18,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 05:29:18,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 05:29:18,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 05:29:18,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 05:29:18,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:29:19,220 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5mjyq87i-media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 05:29:19,221 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o4y5r6fe-media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 05:29:19,224 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/es4sx1bs-media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 05:29:19,227 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ko16iiry-media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 05:29:20,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:23,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:23,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:23,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:24,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:24,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:26,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:30,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:31,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:34,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:34,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:34,208 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:34,210 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:36,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:36,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:36,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:38,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:38,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:38,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:39,228 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:29:41,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:42,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:44,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:44,107 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:44,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:44,109 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:47,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:48,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:48,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:52,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:53,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:53,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:53,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:54,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:54,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:54,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:54,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:54,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:54,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:57,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:00,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:03,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:04,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:04,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:04,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:04,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:06,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:06,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:08,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:08,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:30:08,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:08,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:09,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:30:12,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:14,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:14,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:14,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:14,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:14,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:19,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:19,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:19,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:24,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:30:24,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:24,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:24,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:24,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:24,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:24,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:24,676 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:25,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:25,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:29,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:31,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:35,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:35,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:35,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:35,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:35,704 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:37,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:37,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:39,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:39,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:30:39,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:39,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:30:41,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:43,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:46,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:46,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:46,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:46,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:47,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:49,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:49,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:50,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:50,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:50,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:50,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:52,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:54,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:54,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:55,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:55,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:58,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:03,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:08,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:09,232 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:31:09,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:09,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:09,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:13,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:18,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:23,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:24,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:24,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:24,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:28,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:33,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:38,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:39,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:31:39,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:39,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:39,448 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:43,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:44,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_5c747b5d08a35792803f.png with policy now +2023-07-12 05:31:44,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_8d7535a093e8d31d2071.png with policy now +2023-07-12 05:31:44,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_b02cf0aa477e22c8930d.png with policy now +2023-07-12 05:31:44,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_59c1d57a32cc82a9a3ff.png with policy now +2023-07-12 05:31:45,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:31:45,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:31:45,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:31:45,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:31:45,261 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6hzrl1jo-media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 05:31:45,268 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1eyk0t11-media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 05:31:45,273 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4g9vpns0-media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 05:31:45,276 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k89r9ofs-media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 05:31:48,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:49,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:31:49,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:31:49,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 05:31:49,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 05:31:49,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 05:31:49,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 05:31:49,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:31:49,723 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lo2yfro2-media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 05:31:49,734 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7rk6srp-media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 05:31:49,748 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0b6hbuvr-media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 05:31:49,753 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/98sbv1ki-media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 05:31:53,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:54,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:54,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:54,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:54,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:31:55,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:31:59,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:01,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:04,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:04,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:04,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:04,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:04,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:07,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:07,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:09,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:32:09,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:09,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:09,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:09,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:13,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:14,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:14,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:14,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:14,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:14,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:19,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:19,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:19,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:24,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:24,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:24,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:24,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:24,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:24,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:24,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:25,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:25,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:25,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:30,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:31,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:34,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:34,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:34,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:34,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:36,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:37,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:37,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:39,235 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:32:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:39,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:41,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:43,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:44,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:44,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:44,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:44,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:46,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:49,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:49,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:52,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:54,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:54,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:54,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:55,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:55,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:55,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:55,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:55,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:55,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:57,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:01,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:02,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:05,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:33:05,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:33:05,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:33:05,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:33:07,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:33:07,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:07,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:09,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:33:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:33:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:09,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:13,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:13,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:16,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:33:16,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:33:16,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:33:16,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:33:18,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:19,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:33:19,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:20,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:33:20,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:33:20,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:33:20,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:33:24,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:24,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:24,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:25,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:33:25,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:29,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:34,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:39,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:33:39,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:33:39,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:39,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:42,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:47,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:52,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:54,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:33:54,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:54,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:58,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:03,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:08,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:09,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:34:10,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:10,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:10,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:13,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:15,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_e5cb375d6da828cc3e3e.png with policy now +2023-07-12 05:34:15,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_064f388777b3203b13c5.png with policy now +2023-07-12 05:34:15,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_a24af0c79adf71caef03.png with policy now +2023-07-12 05:34:15,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_9bd0004be4c4a0c4f009.png with policy now +2023-07-12 05:34:15,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:15,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:15,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:15,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:15,404 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z09ayj1o-media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 05:34:15,441 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fc3yp05y-media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 05:34:15,468 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/de38dtsn-media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 05:34:15,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8dzvobj5-media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 05:34:18,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:19,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:19,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:19,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 05:34:19,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 05:34:19,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 05:34:19,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 05:34:19,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:34:20,185 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8e9il45f-media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 05:34:20,186 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ruiyhxl7-media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 05:34:20,209 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b96xpv4i-media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 05:34:20,214 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xh95u0bo-media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 05:34:24,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:24,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:25,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:25,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:25,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:25,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:29,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:31,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:33,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:33,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:33,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:33,634 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:35,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:37,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:37,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:39,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:34:40,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:40,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:40,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:40,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:43,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:43,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:43,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:43,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:43,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:43,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:46,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:49,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:52,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:54,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:54,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:54,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:54,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:55,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:55,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:56,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:56,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:56,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:57,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:01,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:03,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:04,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:04,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:04,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:04,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:07,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:07,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:09,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:35:09,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:11,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:11,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:11,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:35:13,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:14,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:14,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:14,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:14,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:15,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:20,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:20,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:20,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:25,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:25,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:25,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:25,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:26,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:26,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:26,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:27,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:27,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:27,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:35:31,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:32,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:35,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:35,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:35,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:35,520 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:36,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:38,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:38,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:39,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:35:41,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:43,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:43,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:43,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:35:44,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:45,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:45,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:45,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:45,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:46,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:50,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:50,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:50,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:50,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:50,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:50,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:52,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:56,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:57,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:58,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:58,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:58,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:02,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:07,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:09,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:36:13,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:13,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:13,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:36:13,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:18,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:23,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:28,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:28,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:29,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:34,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:39,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:36:40,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:36:43,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:45,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_b1042365de64e0a942b0.png with policy now +2023-07-12 05:36:45,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_dbe013ae177630d5a987.png with policy now +2023-07-12 05:36:45,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_3bca8acf12b4c46ab3ce.png with policy now +2023-07-12 05:36:45,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,103 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_473197dd1f22461890d6.png with policy now +2023-07-12 05:36:45,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:36:45,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:36:45,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:36:45,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:36:45,417 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/trgvsd55-media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 05:36:45,442 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mtidgjlq-media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 05:36:45,448 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n361rlu2-media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 05:36:45,470 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5mzi9b6h-media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 05:36:46,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 05:36:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 05:36:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 05:36:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:36:50,736 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k0i8s6c9-media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 05:36:50,764 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nate0cs8-media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 05:36:50,798 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9gmwzm79-media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 05:36:50,864 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d895tav2-media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 05:36:51,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:55,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:36:56,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:36:57,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:58,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:58,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:02,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:02,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:04,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:04,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:04,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:04,267 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:07,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:08,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:08,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:09,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:37:13,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:13,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:13,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:13,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:13,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:13,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:13,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:14,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:14,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:18,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:20,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:24,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:24,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:24,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:24,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:24,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:26,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:26,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:28,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:28,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:37:28,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:29,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:32,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:35,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:35,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:35,288 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:35,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:35,291 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:38,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:38,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:39,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:37:41,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:44,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:44,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:37:44,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:44,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:45,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:45,860 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:45,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:45,863 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:46,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:50,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:50,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:52,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:55,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:55,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:55,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:55,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:56,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:56,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:57,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:59,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:37:59,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:59,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:02,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:03,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:05,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:38:05,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:38:05,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:38:05,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:38:08,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:38:08,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:09,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:09,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:38:14,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:14,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:38:14,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:38:14,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:14,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:16,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:38:16,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:38:16,671 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:38:16,673 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:38:20,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:20,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:38:20,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:21,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:38:21,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:38:21,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:38:21,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:38:25,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:26,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:38:26,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:29,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:38:29,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:38:29,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:31,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:36,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:39,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:38:42,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:44,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:38:44,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:38:44,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:47,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:52,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:57,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:00,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:00,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:00,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:03,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:08,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:09,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:39:13,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:14,935 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_87ea3e96f26f5ebb57af.png with policy now +2023-07-12 05:39:14,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_ccdaf0fc6a17d6760008.png with policy now +2023-07-12 05:39:14,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_ff1f03141cde2a233297.png with policy now +2023-07-12 05:39:14,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,965 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_a1fff07a952b497ecc10.png with policy now +2023-07-12 05:39:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:15,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:15,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:15,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:15,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:15,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:15,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:15,220 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oicjjrvp-media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 05:39:15,247 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d36xcn2o-media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 05:39:15,255 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/51x666h9-media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 05:39:15,278 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/baq6vdhk-media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 05:39:18,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:20,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:20,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:20,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 05:39:20,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 05:39:20,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 05:39:20,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 05:39:20,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:39:21,218 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sj6f18di-media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 05:39:21,222 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ta3e5y9t-media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 05:39:21,223 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhbaxaw6-media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 05:39:21,229 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zvf2zm7g-media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 05:39:24,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:25,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:26,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:29,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:30,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:30,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:30,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:32,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:35,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:35,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:35,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:35,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:35,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:38,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:38,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:39,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:39:40,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:44,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:45,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:45,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:45,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:45,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:46,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:46,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:46,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:46,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:50,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:50,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:50,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:56,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:56,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:56,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:56,179 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:56,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:56,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:56,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:00,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:00,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:00,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:02,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:06,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:06,197 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:06,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:06,199 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:07,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:08,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:08,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:09,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:40:12,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:14,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:15,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:15,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:15,845 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:16,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:16,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:16,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:16,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:18,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:21,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:21,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:23,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:26,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:26,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:26,336 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:26,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:27,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:27,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:29,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:31,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:31,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:31,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:33,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:34,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:37,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:37,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:37,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:37,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:39,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:39,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:39,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:40:39,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:45,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:45,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:46,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:46,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:46,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:46,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:46,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:46,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:46,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:51,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:51,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:51,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:51,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:51,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:51,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:51,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:57,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:57,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:57,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:41:01,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:01,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:01,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:02,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:07,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:09,255 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:41:13,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:16,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:16,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:16,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:18,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:23,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:28,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:31,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:31,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:31,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:33,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:38,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:39,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:41:44,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:45,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_f98823b83c4cee5e89ac.png with policy now +2023-07-12 05:41:45,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_8b4cb7fa6ec33b1868bc.png with policy now +2023-07-12 05:41:45,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_8fdfffa2228e3683ab99.png with policy now +2023-07-12 05:41:45,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_e32477931d4fd98f823b.png with policy now +2023-07-12 05:41:45,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:41:45,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:41:45,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:41:45,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:41:46,124 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i66oqag7-media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 05:41:46,176 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1wiur99q-media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 05:41:46,177 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k0aqo5fp-media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 05:41:46,179 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dmdilxuj-media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 05:41:46,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:46,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:46,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:50,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:51,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:41:51,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 05:41:51,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 05:41:51,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 05:41:51,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 05:41:51,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:41:51,762 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uk7s2q91-media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 05:41:51,763 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a4q0q80z-media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 05:41:51,769 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r3n8uqve-media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 05:41:51,821 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jvx5xwz6-media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 05:41:56,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:56,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:41:57,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:01,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:01,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:01,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:01,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:03,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:05,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:05,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:05,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:05,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:06,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:09,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:42:09,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:09,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:12,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:15,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:15,431 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:15,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:15,433 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:15,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:15,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:17,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:17,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:17,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:17,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:21,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:22,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:25,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:25,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:25,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:25,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:27,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:27,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:27,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:32,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:32,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:32,843 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:32,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:33,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:36,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:36,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:36,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:36,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:38,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:39,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:42:39,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:39,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:43,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:45,503 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:45,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:45,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:45,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:45,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:48,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:48,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:48,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:48,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:51,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:51,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:53,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:56,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:56,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:56,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:56,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:57,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:57,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:59,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:03,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:03,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:03,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:03,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:05,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:06,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:43:06,468 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:43:06,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:43:06,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:43:09,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:43:09,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:43:09,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:10,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:15,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:16,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:17,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:43:17,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:43:17,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:43:17,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:43:18,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:18,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:18,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:21,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:43:21,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:21,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:22,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:43:22,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:43:22,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:43:22,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:43:27,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:43:27,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:27,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:32,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:33,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:33,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:33,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:38,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:39,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:43:44,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:48,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:48,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:48,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:49,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:54,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:59,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:03,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:03,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:03,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:04,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:09,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:44:10,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:15,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:16,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_49ed245ee23ca275689e.png with policy now +2023-07-12 05:44:16,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_7925401a6a5e9671ca9b.png with policy now +2023-07-12 05:44:16,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_e9a621d16e71d8a70ce6.png with policy now +2023-07-12 05:44:16,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_fea5b00a54b6cde2ffad.png with policy now +2023-07-12 05:44:16,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:16,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:16,739 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:16,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:16,967 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k3fc7efp-media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 05:44:16,977 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vfy9t98q-media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 05:44:16,985 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qvryl6z1-media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 05:44:17,011 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gftkzknp-media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 05:44:18,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:18,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:18,800 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:20,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:21,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:21,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:21,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 05:44:21,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 05:44:21,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 05:44:21,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 05:44:21,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:44:22,342 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/luxrzb9j-media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 05:44:22,359 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ilaz72yt-media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 05:44:22,360 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zrhn2rgh-media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 05:44:22,366 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5xbfyc3e-media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 05:44:26,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:26,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:27,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:31,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:33,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:33,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:33,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:33,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:36,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:36,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:36,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:36,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:36,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:39,261 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:44:39,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:39,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:41,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:45,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:47,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:47,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:47,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:47,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:47,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:48,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:48,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:48,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:51,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:51,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:53,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:57,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:57,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:57,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:57,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:58,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:58,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:58,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:03,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:03,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:45:03,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:03,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:04,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:08,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:08,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:08,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:08,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:09,263 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:45:09,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:10,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:10,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:14,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:16,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:18,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:18,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:18,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:18,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:18,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:18,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:45:18,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:20,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:22,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:22,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:25,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:28,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:28,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:28,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:28,880 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:28,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:33,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:33,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:34,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:34,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:36,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:39,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:45:39,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:39,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:39,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:39,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:40,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:40,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:41,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:46,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:46,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:48,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:45:48,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:48,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:49,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:49,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:49,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:49,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:51,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:52,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:52,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:54,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:54,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:54,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:54,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:57,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:58,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:58,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:46:02,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:03,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:03,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:03,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:08,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:09,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:46:13,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:18,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:19,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:19,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:19,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:24,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:29,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:34,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:34,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:34,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:34,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:39,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:46:39,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:44,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:48,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_482ad8df2ce841450721.png with policy now +2023-07-12 05:46:48,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_4e545b1d5aa48bae02e7.png with policy now +2023-07-12 05:46:48,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_d6a886a91a9a1c97ff8e.png with policy now +2023-07-12 05:46:48,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_e377b1a6785b8362f8b8.png with policy now +2023-07-12 05:46:48,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:46:48,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:46:48,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:46:48,617 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:46:48,840 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/98s6nbmv-media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 05:46:48,869 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zssq4f35-media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 05:46:48,883 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/foqgvxvd-media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 05:46:49,070 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rauoneb8-media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 05:46:49,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:49,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:49,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:50,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:52,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:46:52,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 05:46:52,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 05:46:52,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 05:46:52,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 05:46:52,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:46:52,806 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zwzkrytu-media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 05:46:52,820 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/na7ko6dv-media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 05:46:52,824 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/39810n4h-media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 05:46:52,827 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0pnkliau-media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 05:46:56,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:58,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:46:58,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:01,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:04,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:04,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:04,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:04,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:06,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:08,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:08,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:08,132 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:08,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:09,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:47:10,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:10,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:12,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:16,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:17,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:17,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:17,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:17,584 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:18,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:19,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:19,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:19,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:22,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:22,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:24,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:28,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:28,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:28,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:28,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:28,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:28,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:30,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:34,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:36,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:36,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:36,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:39,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:47:39,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:39,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:39,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:39,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:40,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:40,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:40,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:46,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:46,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:48,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:48,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:48,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:48,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:51,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:51,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:51,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:51,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:52,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:52,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:57,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:58,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:58,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:58,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:58,542 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:58,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:58,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:02,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:04,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:06,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:06,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:06,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:07,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:08,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:48:08,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:48:08,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:48:08,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:48:09,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:48:10,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:48:10,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:12,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:16,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:18,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:19,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:48:19,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:48:19,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:48:19,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:48:21,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:21,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:21,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:22,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:48:22,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:24,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:24,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:48:24,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:48:24,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:48:24,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:48:28,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:48:28,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:29,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:34,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:36,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:36,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:36,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:39,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:48:40,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:45,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:50,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:51,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:51,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:51,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:55,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:00,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:05,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:06,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:06,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:06,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:09,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:49:11,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:16,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:18,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_ba0b7fdc45db85580b1b.png with policy now +2023-07-12 05:49:18,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_d46d1a613eb099818463.png with policy now +2023-07-12 05:49:18,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_18b8edd8e82489c2e700.png with policy now +2023-07-12 05:49:18,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_b7ab393ffe13b943540d.png with policy now +2023-07-12 05:49:18,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:18,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:18,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:18,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:18,483 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/unyb8m4b-media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 05:49:18,529 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d9ykmmtt-media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 05:49:18,561 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j5dvpws8-media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 05:49:18,604 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/roa0epyo-media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 05:49:21,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:21,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:21,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:21,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:22,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:22,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:22,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 05:49:22,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 05:49:22,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 05:49:22,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 05:49:22,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:49:23,334 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pw01x64b-media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 05:49:23,341 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xj3u4pzm-media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 05:49:23,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27rl1os2-media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 05:49:23,359 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/im67pzt9-media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 05:49:27,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:28,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:29,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:33,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:35,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:36,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:36,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:36,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:38,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:38,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:38,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:38,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:39,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:39,273 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:49:41,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:41,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:44,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:47,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:47,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:47,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:47,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:47,746 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:49,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:52,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:52,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:52,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:53,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:53,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:55,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:58,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:58,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:58,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:58,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:59,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:59,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:00,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:05,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:06,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:08,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:08,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:08,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:09,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:50:09,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:09,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:09,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:09,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:11,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:11,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:11,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:16,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:17,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:19,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:19,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:19,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:19,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:21,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:23,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:23,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:23,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:23,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:23,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:26,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:29,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:29,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:29,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:29,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:29,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:32,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:35,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:35,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:38,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:38,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:38,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:38,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:39,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:39,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:39,155 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:39,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:39,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:50:41,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:41,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:44,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:47,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:48,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:48,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:48,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:48,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:49,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:53,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:53,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:53,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:53,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:53,308 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:53,310 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:53,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:53,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:53,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:54,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:59,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:59,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:59,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:04,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:08,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:08,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:08,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:09,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:51:10,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:15,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:20,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:23,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:23,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:23,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:25,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:30,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:35,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:39,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:39,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:39,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:39,981 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:51:40,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:45,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:47,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_2048ef7db009bfd7c05d.png with policy now +2023-07-12 05:51:47,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,750 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_02fa9f2ed8ff12c99aa7.png with policy now +2023-07-12 05:51:47,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_fb6772a80e5d23777bc1.png with policy now +2023-07-12 05:51:47,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_506d872797968a090e00.png with policy now +2023-07-12 05:51:47,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:51:47,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:51:47,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:51:47,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:51:48,043 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bp7aptio-media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 05:51:48,045 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r7xipqxk-media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 05:51:48,058 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7z3iszbl-media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 05:51:48,098 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5p3dn42s-media/images/validation_1390_506d872797968a090e00.png +2023-07-12 05:51:51,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:53,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:51:53,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:51:53,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 05:51:53,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_506d872797968a090e00.png +2023-07-12 05:51:53,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 05:51:53,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 05:51:53,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:51:53,836 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/54bnzhan-media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 05:51:53,842 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2en5mv0o-media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 05:51:53,843 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nu9csk0j-media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 05:51:53,845 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/70rcr4ch-media/images/validation_1390_506d872797968a090e00.png +2023-07-12 05:51:54,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:54,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:54,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:56,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:58,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:51:59,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:01,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:05,503 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:07,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:07,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:07,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:07,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:07,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:09,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:52:09,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:09,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:09,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:11,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:11,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:12,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:17,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:17,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:17,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:17,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:17,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:17,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:17,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:22,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:23,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:24,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:24,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:24,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:26,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:26,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:26,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:26,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:28,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:29,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:29,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:33,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:35,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:36,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:36,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:36,828 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:36,830 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:38,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:39,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:52:39,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:39,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:39,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:41,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:41,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:44,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:47,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:47,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:47,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:47,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:47,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:47,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:49,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:53,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:54,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:54,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:54,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:54,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:57,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:57,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:57,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:57,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:59,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:59,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:00,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:05,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:06,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:07,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:53:07,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:53:07,444 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:53:07,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:53:09,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:53:09,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:09,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:09,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:53:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:12,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:17,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:17,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:53:17,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:53:17,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:53:17,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:53:17,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:53:17,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:22,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:53:22,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:22,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:53:22,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:53:22,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:53:23,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:53:23,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:24,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:24,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:24,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:27,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:29,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:32,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:37,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:39,283 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:53:39,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:39,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:39,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:42,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:47,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:52,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:54,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:54,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:54,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:58,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:03,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:08,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:09,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:54:09,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:09,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:09,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:13,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:16,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,769 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_66aa1d0173b44509cd1d.png with policy now +2023-07-12 05:54:16,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_e84af5fa7376533300be.png with policy now +2023-07-12 05:54:16,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,810 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_f29eae3956b8b680487b.png with policy now +2023-07-12 05:54:16,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_d020640095c66cf43293.png with policy now +2023-07-12 05:54:16,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:16,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:16,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:16,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:54:17,080 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cvyixbfn-media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 05:54:17,101 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqffnhpz-media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 05:54:17,139 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t40ljbf4-media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 05:54:17,149 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r2n3m1j1-media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 05:54:18,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:54:18,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 05:54:18,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 05:54:18,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 05:54:18,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 05:54:18,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:54:18,374 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mr2fk0wv-media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 05:54:18,393 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/so5kups0-media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 05:54:18,394 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gf9js64d-media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 05:54:18,400 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/694at0by-media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 05:54:18,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:23,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:24,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:24,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:24,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:24,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:26,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:29,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:30,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:34,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:35,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:35,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:35,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:35,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:54:36,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:54:36,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:39,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:54:40,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:40,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:40,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:40,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:42,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:45,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:46,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:46,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:46,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:46,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:54:48,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:54:48,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:51,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:54,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:56,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:56,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:56,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:56,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:56,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:56,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:56,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:56,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:00,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:00,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:02,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:06,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:07,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:07,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:07,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:07,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:07,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:09,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:55:12,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:12,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:12,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:12,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:12,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:12,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:17,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:17,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:17,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:17,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:17,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:18,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:18,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:23,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:24,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:27,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:27,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:27,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:27,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:27,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:27,778 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:27,780 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:29,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:30,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:30,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:34,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:36,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:37,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:37,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:37,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:37,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:39,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:55:39,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:42,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:42,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:42,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:42,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:42,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:45,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:48,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:48,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:48,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:48,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:48,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:48,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:51,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:52,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:52,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:52,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:52,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:54,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:54,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:57,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:57,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:57,525 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:57,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:00,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:56:02,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:07,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:09,288 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:56:12,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:12,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:12,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:12,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:17,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:22,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:27,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:27,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:27,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:27,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:32,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:37,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:39,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:56:42,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:42,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:42,961 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:42,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:47,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,126 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_89c2655d6ea12bc678a5.png with policy now +2023-07-12 05:56:47,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_754b2c1e14f253b4c743.png with policy now +2023-07-12 05:56:47,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_0de4ce377ce0650003a1.png with policy now +2023-07-12 05:56:47,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_57eaa364b09be7b1c708.png with policy now +2023-07-12 05:56:47,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:56:47,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:56:47,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:56:47,208 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:56:47,443 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/si3snp5r-media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 05:56:47,474 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3z1auwza-media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 05:56:47,499 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5x8c3jho-media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 05:56:47,504 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3j49o5ps-media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 05:56:48,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:48,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:56:48,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 05:56:48,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 05:56:48,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 05:56:48,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 05:56:48,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:56:49,306 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5zk2sxzl-media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 05:56:49,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gth5ewve-media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 05:56:49,354 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ywuw4uk-media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 05:56:49,462 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3zjuqz6b-media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 05:56:53,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:54,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:56:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:56:58,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:58,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:58,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:58,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:00,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:03,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:06,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:06,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:06,427 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:06,429 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:06,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:06,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:09,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:09,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:57:12,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:13,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:57:13,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:13,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:57:14,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:17,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:17,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:17,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:17,026 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:18,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:18,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:19,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:24,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:25,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:27,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:27,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:27,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:27,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:28,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:28,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:57:30,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:30,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:30,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:35,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:36,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:38,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:38,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:38,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:38,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:39,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:57:41,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:42,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:42,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:43,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:43,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:57:43,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:57:46,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:48,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:48,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:48,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:48,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:49,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:49,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:52,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:55,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:57,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:58,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:58,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:58,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:58,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:58,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:58,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:01,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:58:01,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:03,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:07,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:08,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:58:08,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:08,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:58:08,613 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:58:08,615 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:58:09,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:58:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:58:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:13,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:13,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:13,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:19,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:19,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:58:19,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:58:19,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:19,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:58:19,374 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:58:24,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:58:24,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:58:24,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:58:24,018 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:58:24,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:25,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:58:25,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:28,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:28,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:29,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:34,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:39,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:58:40,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:43,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:58:43,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:43,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:46,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:51,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:56,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:58,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:58:58,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:58,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:02,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:07,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:09,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:59:12,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:13,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:14,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:14,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:18,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:18,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_65621df76009498b6856.png with policy now +2023-07-12 05:59:18,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_82f075e0330d5c29720e.png with policy now +2023-07-12 05:59:18,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_8760dbb0b29df7f09a82.png with policy now +2023-07-12 05:59:18,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_51314bed48a62a63179f.png with policy now +2023-07-12 05:59:18,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:18,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:18,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:18,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:18,841 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9b30smef-media/images/validation_1420_65621df76009498b6856.png +2023-07-12 05:59:18,887 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/75xlacfg-media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 05:59:18,905 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3khyp1e-media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 05:59:19,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:59:19,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_65621df76009498b6856.png +2023-07-12 05:59:19,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 05:59:19,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 05:59:19,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 05:59:19,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:59:19,448 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/10x4mewm-media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 05:59:19,733 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eucjy52z-media/images/validation_1420_65621df76009498b6856.png +2023-07-12 05:59:19,752 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6pk8oa2q-media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 05:59:19,762 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1xgxo4z-media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 05:59:19,786 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/50w60ak1-media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 05:59:23,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:25,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:28,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:28,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:28,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:28,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:29,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:31,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:34,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:37,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:37,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:37,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:37,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:37,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:59:37,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:39,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:59:40,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:43,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:44,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:44,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:44,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:46,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:46,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:46,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:46,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:46,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:49,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:59:49,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:52,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:55,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:58,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:58,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:58,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:58,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:58,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:59,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:59,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:59,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:01,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:01,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:03,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:07,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:09,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:00:09,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:09,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:09,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:09,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:09,512 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:13,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:13,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:14,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:00:14,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:14,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:15,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:19,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:19,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:19,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:19,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:19,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:20,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:25,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:25,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:25,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:29,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:00:29,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:29,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:29,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:29,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:29,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:29,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:30,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:31,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:31,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:35,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:37,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:39,300 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:00:39,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:39,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:39,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:39,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:41,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:43,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:43,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:44,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:44,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:47,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:49,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:50,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:50,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:50,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:50,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:52,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:54,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:54,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:54,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:54,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:55,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:55,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:58,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:59,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:59,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:01,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:01:04,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:09,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:09,300 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:01:14,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:14,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:14,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:14,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:19,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:24,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:29,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:29,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:29,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:30,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:35,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:39,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:01:41,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:44,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:44,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:44,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:46,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:49,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,246 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_a5e480b9cae1eda40b25.png with policy now +2023-07-12 06:01:49,255 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_6fd754942b22d95c6796.png with policy now +2023-07-12 06:01:49,266 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_bb8a00b843066df3ae5d.png with policy now +2023-07-12 06:01:49,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_acb50883d3668759985a.png with policy now +2023-07-12 06:01:49,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:01:49,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:01:49,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:01:49,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:01:49,541 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fq0jam1h-media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 06:01:49,556 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1phq2vjm-media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 06:01:49,590 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5jidxzj7-media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 06:01:49,599 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fb50ghw2-media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 06:01:49,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:01:49,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 06:01:49,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 06:01:49,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 06:01:49,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 06:01:49,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:01:50,301 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bb1zic44-media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 06:01:50,311 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l6a2yr5w-media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 06:01:50,322 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r5mektti-media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 06:01:50,324 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lug8f6id-media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 06:01:52,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:55,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:01:57,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:58,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:01:59,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:59,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:59,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:01,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:03,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:08,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:08,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:08,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:08,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:08,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:09,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:09,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:02:14,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:14,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:14,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:14,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:02:14,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:02:14,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:17,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:17,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:17,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:17,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:20,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:20,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:20,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:25,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:26,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:28,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:28,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:28,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:28,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:29,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:02:29,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:02:29,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:31,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:32,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:32,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:36,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:38,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:38,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:38,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:38,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:38,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:39,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:02:41,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:44,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:44,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:45,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:02:45,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:02:45,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:46,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:48,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:48,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:48,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:48,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:50,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:50,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:51,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:56,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:56,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:58,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:58,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:58,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:58,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:00,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:00,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:00,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:02,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:02,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:02,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:07,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:08,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:09,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:03:09,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:03:09,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:03:09,052 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:09,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:03:12,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:14,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:14,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:15,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:15,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:15,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:18,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:18,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:03:18,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:03:18,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:03:18,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:20,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:20,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:23,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:23,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:03:23,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:03:23,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:03:23,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:26,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:26,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:29,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:30,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:30,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:30,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:34,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:39,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:03:40,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:45,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:45,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:46,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:46,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:51,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:56,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:00,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:00,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:00,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:02,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:07,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:09,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:04:12,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:16,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:16,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:16,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:18,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:18,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_ff68423dd6be58cda77c.png with policy now +2023-07-12 06:04:18,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,290 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_7456d4bbc7e1692f0395.png with policy now +2023-07-12 06:04:18,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_2d1c628ad6cab3a92695.png with policy now +2023-07-12 06:04:18,312 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,312 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_84ea7cea928542cee5c3.png with policy now +2023-07-12 06:04:18,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:18,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:18,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:18,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:04:18,568 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s000v0hj-media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 06:04:18,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hauo85gy-media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 06:04:18,599 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3wbj8kwd-media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 06:04:18,612 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1cmlb1rf-media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 06:04:20,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:04:20,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 06:04:20,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 06:04:20,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 06:04:20,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 06:04:20,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:04:20,816 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ojkoaewj-media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 06:04:20,836 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/16abjk25-media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 06:04:20,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v1xy8c1k-media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 06:04:20,879 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ryac08n9-media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 06:04:23,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:26,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:28,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:29,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:31,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:31,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:31,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:32,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:34,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:37,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:37,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:37,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:37,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:04:38,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:04:38,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:39,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:04:40,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:44,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:46,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:47,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:47,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:47,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:47,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:47,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:47,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:47,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:04:50,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:04:50,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:51,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:56,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:56,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:58,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:58,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:58,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:58,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:02,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:02,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:02,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:02,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:02,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:02,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:07,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:08,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:09,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:09,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:09,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:09,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:09,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:05:13,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:14,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:14,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:18,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:18,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:18,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:18,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:18,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:18,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:18,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:18,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:20,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:20,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:23,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:26,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:28,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:28,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:28,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:28,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:29,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:32,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:32,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:33,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:33,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:33,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:34,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:38,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:39,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:39,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:39,244 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:39,246 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:39,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:05:40,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:44,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:44,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:45,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:48,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:48,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:48,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:49,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:49,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:49,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:49,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:50,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:50,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:51,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:53,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:53,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:53,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:53,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:56,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:56,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:56,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:06:01,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:03,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:07,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:09,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:06:13,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:18,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:18,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:18,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:18,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:23,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:28,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:33,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:33,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:33,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:34,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:39,311 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:06:40,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:45,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:48,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:48,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:48,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:48,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_6eeb827ca7d660925ab1.png with policy now +2023-07-12 06:06:48,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_1c96b81934ff3d0e0e7a.png with policy now +2023-07-12 06:06:48,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_f51ce3f90deb8ec9156b.png with policy now +2023-07-12 06:06:48,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,684 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_4fbcb05acbe8b16ae524.png with policy now +2023-07-12 06:06:48,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:06:48,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:06:48,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:06:48,727 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:06:48,965 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/feae43rl-media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 06:06:48,978 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t95a36dv-media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 06:06:48,992 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zur88039-media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 06:06:49,059 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d8glq9b5-media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 06:06:50,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:51,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:06:51,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 06:06:51,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 06:06:51,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 06:06:51,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 06:06:51,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:06:51,421 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2dk3b1ob-media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 06:06:51,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c3sbk92l-media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 06:06:51,455 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4r8erk3-media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 06:06:51,480 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/st813tzh-media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 06:06:56,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:57,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:06:58,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:01,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:03,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:03,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:08,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:08,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:09,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:09,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:09,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:09,477 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:09,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:07:15,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:15,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:15,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:18,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:18,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:18,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:18,184 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:18,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:18,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:18,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:20,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:21,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:21,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:25,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:27,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:28,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:28,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:28,299 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:28,301 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:30,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:33,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:33,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:33,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:36,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:38,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:38,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:38,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:38,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:39,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:39,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:39,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:07:42,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:45,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:48,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:48,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:49,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:49,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:49,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:51,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:51,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:53,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:57,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:58,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:59,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:59,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:59,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:59,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:03,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:03,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:03,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:03,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:03,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:03,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:09,315 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:08:09,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:09,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:09,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:08:09,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:08:09,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:08:09,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:14,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:15,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:15,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:18,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:18,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:18,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:19,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:08:19,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:08:19,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:08:19,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:20,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:21,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:21,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:24,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:08:24,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:08:24,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:08:24,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:25,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:27,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:27,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:30,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:34,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:34,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:34,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:36,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:39,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:08:41,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:46,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:49,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:49,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:49,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:52,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:57,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:02,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:04,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:04,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:04,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:07,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:09,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:09:12,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:17,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:18,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,790 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_fc2c6a80029568453397.png with policy now +2023-07-12 06:09:18,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_e0e86f6d8760c4a8cb7e.png with policy now +2023-07-12 06:09:18,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_bafecf6d5c7a9080d983.png with policy now +2023-07-12 06:09:18,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_688928104b18c14855ba.png with policy now +2023-07-12 06:09:18,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:18,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:18,890 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:18,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:09:19,116 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d6d4yzq2-media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 06:09:19,123 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ixw92z9-media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 06:09:19,137 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0b8nxsht-media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 06:09:19,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:19,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:19,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:19,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x4k7nj3a-media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 06:09:21,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:09:21,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 06:09:21,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 06:09:21,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 06:09:21,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 06:09:21,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:09:21,979 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ttix2yx5-media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 06:09:21,984 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3fedqpvk-media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 06:09:22,036 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0on5etyj-media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 06:09:22,046 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ollxezcs-media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 06:09:22,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:27,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:27,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:27,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:33,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:33,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:34,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:34,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:34,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:37,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:37,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:37,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:37,588 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:09:38,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:39,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:09:39,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:09:39,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:43,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:45,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:48,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:48,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:48,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:48,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:09:49,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:49,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:49,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:49,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:51,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:09:51,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:54,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:57,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:58,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:58,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:58,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:58,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:00,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:03,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:03,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:04,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:04,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:04,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:05,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:09,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:10:09,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:09,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:09,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:09,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:09,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:09,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:10,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:15,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:16,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:19,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:19,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:19,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:19,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:19,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:19,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:19,379 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:21,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:21,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:21,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:26,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:27,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:29,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:29,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:29,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:29,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:32,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:33,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:33,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:34,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:34,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:34,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:37,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:39,319 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:10:39,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:39,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:39,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:39,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:39,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:39,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:43,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:45,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:49,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:49,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:49,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:49,054 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:49,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:49,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:49,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:49,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:51,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:51,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:53,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:53,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:53,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:53,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:54,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:57,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:57,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:59,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:04,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:11:04,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:04,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:05,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:09,321 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:11:11,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:16,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:19,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:11:19,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:19,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:21,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:26,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:31,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:34,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:34,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:11:34,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:37,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:39,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:11:43,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:48,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_70617e686456e8dfd42a.png with policy now +2023-07-12 06:11:48,313 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,313 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_724c86f275815432b7e4.png with policy now +2023-07-12 06:11:48,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,314 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_841f2b0f1c0c1e17dd9d.png with policy now +2023-07-12 06:11:48,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,314 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_d752e1cd90b5707c0547.png with policy now +2023-07-12 06:11:48,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:11:48,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:11:48,348 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:11:48,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:48,349 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:11:48,579 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c9eh6n8h-media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 06:11:48,627 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hskj0hlt-media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 06:11:48,628 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hqjfwq55-media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 06:11:48,657 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ccgqw1h-media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 06:11:49,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:49,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:52,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:11:52,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 06:11:52,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 06:11:52,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 06:11:52,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 06:11:52,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:11:52,546 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pvv95184-media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 06:11:52,561 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hbc8avim-media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 06:11:52,633 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tzxznh8a-media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 06:11:52,649 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7swltc99-media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 06:11:53,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:57,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:11:58,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:11:58,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:04,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:04,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:04,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:04,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:07,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:07,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:07,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:07,333 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:09,323 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:12:10,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:10,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:10,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:16,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:16,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:16,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:16,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:16,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:16,998 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:19,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:19,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:12:19,447 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:21,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:22,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:22,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:26,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:27,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:27,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:27,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:27,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:28,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:28,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:32,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:34,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:35,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:12:35,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:35,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:37,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:38,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:38,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:38,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:38,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:39,325 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:12:40,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:40,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:43,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:46,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:48,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:49,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:49,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:49,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:49,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:50,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:12:50,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:50,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:54,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:58,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:58,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:58,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:58,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:58,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:59,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:04,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:04,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:05,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:05,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:05,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:05,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:08,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:13:08,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:13:08,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:13:08,859 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:13:09,325 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:13:10,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:10,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:10,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:15,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:16,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:19,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:13:19,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:13:19,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:13:19,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:13:21,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:21,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:21,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:21,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:22,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:22,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:23,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:13:23,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:13:23,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:13:23,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:13:26,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:31,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:36,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:36,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:36,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:37,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:39,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:13:42,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:47,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:51,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:51,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:51,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:53,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:58,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:03,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:06,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:06,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:06,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:08,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:09,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:14:13,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:18,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:18,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_a4d38678421736956450.png with policy now +2023-07-12 06:14:18,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_22921140211d309efc15.png with policy now +2023-07-12 06:14:18,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_577a6eadd1895fbf8edd.png with policy now +2023-07-12 06:14:18,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,617 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_17a3ddb8f942d0a4a42c.png with policy now +2023-07-12 06:14:18,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:18,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:18,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:18,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:14:18,891 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zvenh6eb-media/images/validation_1480_22921140211d309efc15.png +2023-07-12 06:14:18,927 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8uejm277-media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 06:14:18,933 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fhw63gwo-media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 06:14:18,936 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlkf1ued-media/images/validation_1480_a4d38678421736956450.png +2023-07-12 06:14:21,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:21,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:21,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:22,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:14:22,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:22,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_a4d38678421736956450.png +2023-07-12 06:14:22,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 06:14:22,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_22921140211d309efc15.png +2023-07-12 06:14:22,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 06:14:22,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:14:23,152 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5i48pp2m-media/images/validation_1480_22921140211d309efc15.png +2023-07-12 06:14:23,161 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/579nxgn4-media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 06:14:23,162 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a26fke29-media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 06:14:23,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:23,496 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/93wdtvuc-media/images/validation_1480_a4d38678421736956450.png +2023-07-12 06:14:28,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:28,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:28,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:33,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:34,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:36,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:36,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:36,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:38,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:38,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:38,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:38,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:38,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:14:39,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:14:40,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:14:40,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:44,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:46,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:48,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:48,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:48,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:48,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:14:50,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:51,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:51,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:51,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:52,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:14:52,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:56,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:58,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:59,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:59,070 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:59,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:59,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:01,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:05,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:05,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:06,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:06,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:06,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:08,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:08,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:08,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:08,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:09,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:15:11,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:11,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:11,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:17,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:17,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:19,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:19,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:19,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:19,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:21,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:15:21,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:21,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:22,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:23,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:23,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:28,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:29,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:30,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:30,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:30,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:30,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:33,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:35,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:35,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:37,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:15:37,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:37,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:38,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:39,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:15:40,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:40,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:40,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:40,961 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:41,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:41,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:44,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:47,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:50,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:50,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:50,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:50,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:50,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:52,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:15:52,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:52,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:53,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:53,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:55,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:55,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:55,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:55,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:55,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:59,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:59,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:16:00,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:05,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:07,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:07,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:07,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:09,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:16:11,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:16,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:21,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:22,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:22,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:22,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:26,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:31,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:36,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:37,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:37,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:37,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:39,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:16:42,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:47,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:49,751 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,751 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_0084cd73f87c0d0b983c.png with policy now +2023-07-12 06:16:49,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,761 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_5443848005d4e6d0afaa.png with policy now +2023-07-12 06:16:49,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_4b4b982230746456caa0.png with policy now +2023-07-12 06:16:49,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_09e2e8132566f85bd196.png with policy now +2023-07-12 06:16:49,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:16:49,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:16:49,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:16:49,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:16:50,046 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8pjhc7sa-media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 06:16:50,062 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c2euj2gi-media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 06:16:50,087 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ouaf2f4-media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 06:16:50,088 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u2tkuufp-media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 06:16:52,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:52,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:52,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:52,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:53,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:16:53,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:16:53,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 06:16:53,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 06:16:53,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 06:16:53,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 06:16:53,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:16:53,815 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vcd954ma-media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 06:16:53,829 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sryu0cre-media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 06:16:53,832 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v9cb45fr-media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 06:16:53,842 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y0lcrnki-media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 06:16:58,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:59,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:00,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:03,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:06,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:07,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:07,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:07,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:08,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:09,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:17:10,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:10,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:10,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:10,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:13,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:13,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:14,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:20,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:20,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:20,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:20,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:20,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:20,887 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:22,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:22,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:22,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:25,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:27,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:27,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:31,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:31,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:31,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:31,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:31,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:34,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:34,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:36,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:38,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:38,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:38,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:39,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:17:41,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:41,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:41,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:41,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:41,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:41,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:42,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:48,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:48,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:51,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:51,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:51,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:51,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:53,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:53,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:53,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:53,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:55,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:55,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:58,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:02,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:02,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:02,082 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:02,084 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:02,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:02,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:04,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:08,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:08,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:08,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:09,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:09,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:18:09,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:12,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:12,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:12,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:12,254 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:14,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:16,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:16,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:19,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:22,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:22,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:22,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:22,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:23,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:23,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:24,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:24,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:24,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:24,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:27,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:27,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:27,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:27,176 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:29,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:30,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:30,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:34,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:39,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:18:39,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:39,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:39,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:40,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:45,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:50,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:54,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:54,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:54,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:55,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:00,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:05,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:09,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:19:09,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:09,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:09,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:10,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:15,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:20,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:21,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_c6e34c37f93334a75319.png with policy now +2023-07-12 06:19:21,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_de9f5fe5746030c6a74f.png with policy now +2023-07-12 06:19:21,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_cfb49806a7152804aa8b.png with policy now +2023-07-12 06:19:21,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_9a03a9d55b0a9f41b234.png with policy now +2023-07-12 06:19:22,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:22,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:19:22,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:19:22,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:19:22,301 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/euqe6m5o-media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 06:19:22,327 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/53lw2ti7-media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 06:19:22,344 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n1qw7ry6-media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 06:19:22,362 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qwq9mtlj-media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 06:19:24,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:24,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:24,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:26,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:27,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:19:27,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 06:19:27,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 06:19:27,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 06:19:27,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 06:19:27,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:19:27,482 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lb1rqtn9-media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 06:19:27,492 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7fcsfs6r-media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 06:19:27,503 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8jwnbc9d-media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 06:19:27,514 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5goed0qy-media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 06:19:32,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:32,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:34,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:37,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:39,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:19:39,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:39,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:39,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:41,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:41,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:41,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:19:41,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:19:41,253 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:19:43,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:48,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:19:48,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:48,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:51,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:51,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:19:51,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:19:51,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:19:53,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:54,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:54,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:54,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:55,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:19:55,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:58,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:02,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:02,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:02,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:02,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:02,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:02,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:04,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:09,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:09,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:20:09,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:09,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:09,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:09,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:11,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:11,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:11,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:12,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:14,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:16,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:16,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:20,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:22,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:22,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:22,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:22,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:23,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:23,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:24,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:24,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:24,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:25,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:30,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:30,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:32,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:32,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:32,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:32,374 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:36,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:37,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:37,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:39,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:20:39,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:39,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:39,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:41,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:42,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:42,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:42,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:42,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:44,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:44,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:47,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:51,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:52,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:52,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:52,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:52,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:52,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:54,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:54,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:54,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:57,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:57,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:57,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:57,587 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:57,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:58,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:58,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:21:03,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:05,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:21:08,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:09,344 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:21:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:09,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:09,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:21:13,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:18,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:23,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:24,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:24,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:24,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:21:29,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:34,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:39,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:39,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:21:40,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:40,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:40,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:21:44,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:49,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:51,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_be748917843477368009.png with policy now +2023-07-12 06:21:51,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_ec02c5d5a2c3f65267f6.png with policy now +2023-07-12 06:21:51,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_d4a2e70a118507b473e4.png with policy now +2023-07-12 06:21:51,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_11ba5556c102236aa637.png with policy now +2023-07-12 06:21:51,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:21:51,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:21:51,984 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:21:51,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:21:52,202 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ca20jn28-media/images/validation_1510_be748917843477368009.png +2023-07-12 06:21:52,225 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qdf8ditg-media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 06:21:52,232 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n9yjifrb-media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 06:21:52,249 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iyqwlfq5-media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 06:21:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:21:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 06:21:54,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 06:21:54,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 06:21:54,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_be748917843477368009.png +2023-07-12 06:21:54,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:21:54,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:54,908 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kokerfdz-media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 06:21:54,918 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/68uy5ee7-media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 06:21:54,924 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2z7g3bep-media/images/validation_1510_be748917843477368009.png +2023-07-12 06:21:54,927 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blwj50d1-media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 06:21:55,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:55,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:55,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:00,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:01,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:01,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:05,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:08,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:09,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:22:10,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:10,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:10,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:11,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:11,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:11,321 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:11,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:11,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:15,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:15,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:17,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:21,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:21,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:21,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:21,816 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:22,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:26,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:26,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:26,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:28,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:29,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:32,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:32,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:32,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:32,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:33,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:36,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:36,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:39,348 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:22:39,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:41,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:41,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:41,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:42,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:42,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:42,137 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:42,139 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:43,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:43,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:44,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:49,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:50,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:52,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:52,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:52,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:52,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:54,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:57,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:57,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:57,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:57,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:57,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:00,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:02,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:02,767 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:02,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:02,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:04,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:04,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:05,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:09,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:23:10,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:11,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:12,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:12,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:12,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:12,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:12,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:12,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:12,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:15,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:18,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:18,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:21,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:22,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:22,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:22,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:22,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:25,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:25,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:26,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:27,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:27,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:27,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:27,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:27,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:27,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:27,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:31,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:33,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:33,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:36,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:39,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:23:42,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:42,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:42,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:42,620 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:47,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:52,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:57,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:57,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:57,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:58,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:03,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:08,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:09,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:24:12,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:24:12,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:12,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:24:13,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:18,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:21,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_3d2b9accd30d9956baeb.png with policy now +2023-07-12 06:24:21,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_634e550898bfbdb9ac9f.png with policy now +2023-07-12 06:24:21,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_790fbc2ddb08d6dd0d88.png with policy now +2023-07-12 06:24:21,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,395 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_5140a0409a56d2f968a2.png with policy now +2023-07-12 06:24:21,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:21,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:24:21,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:24:21,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:24:21,684 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/suof8yrt-media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 06:24:21,696 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjcv0qpf-media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 06:24:21,720 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cspxx5or-media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 06:24:21,830 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qavi2ft6-media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 06:24:22,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:24:22,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 06:24:22,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 06:24:22,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 06:24:22,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 06:24:22,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:24:22,570 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ranvhjui-media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 06:24:22,572 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vkrcq6yg-media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 06:24:22,575 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v7ddkiw4-media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 06:24:22,663 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tqt1nhuw-media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 06:24:24,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:27,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:27,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:24:27,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:24:29,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:29,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:35,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:36,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:39,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:24:40,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:41,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:41,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:24:41,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:24:41,053 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:24:42,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:42,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:24:43,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:24:43,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:46,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:50,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:50,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:50,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:24:50,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:24:50,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:24:51,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:56,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:57,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:24:57,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:57,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:57,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:00,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:00,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:00,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:00,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:01,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:04,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:04,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:06,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:09,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:25:11,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:11,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:11,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:11,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:11,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:11,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:12,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:13,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:13,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:13,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:18,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:18,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:20,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:20,976 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:20,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:20,978 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:23,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:25,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:25,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:28,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:28,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:28,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:29,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:31,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:31,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:31,051 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:31,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:32,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:32,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:35,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:39,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:25:39,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:40,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:41,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:41,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:41,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:41,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:43,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:45,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:46,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:51,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:52,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:52,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:52,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:52,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:53,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:53,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:56,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:57,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:57,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:57,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:57,123 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:58,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:58,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:58,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:00,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:26:00,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:26:02,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:07,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:09,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:26:13,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:13,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:13,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:26:13,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:18,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:23,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:28,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:28,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:28,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:33,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:38,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:39,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:26:43,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:26:43,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:43,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:43,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:48,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:51,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_c983d04e3b22e1d9100a.png with policy now +2023-07-12 06:26:51,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,565 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_91df6d540afc90aa229c.png with policy now +2023-07-12 06:26:51,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_7e2daf917c565f387c73.png with policy now +2023-07-12 06:26:51,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_6550992879b720b36137.png with policy now +2023-07-12 06:26:51,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:26:51,625 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:26:51,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:26:51,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:26:51,866 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8e4y9sja-media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 06:26:51,874 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i1uwyt11-media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 06:26:51,889 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/obqfolwo-media/images/validation_1530_6550992879b720b36137.png +2023-07-12 06:26:51,902 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xug1yaoa-media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 06:26:53,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:56,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:26:56,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:26:56,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 06:26:56,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_6550992879b720b36137.png +2023-07-12 06:26:56,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 06:26:56,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 06:26:56,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:26:57,097 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uyv2o5hh-media/images/validation_1530_6550992879b720b36137.png +2023-07-12 06:26:57,104 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5bcoyjf8-media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 06:26:57,110 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/82y3z0rh-media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 06:26:57,110 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pflkmxd2-media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 06:26:58,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:26:58,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:58,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:58,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:01,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:03,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:03,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:09,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:09,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:27:10,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:11,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:11,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:11,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:11,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:13,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:13,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:13,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:27:14,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:17,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:17,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:20,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:21,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:21,853 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:21,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:21,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:24,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:24,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:26,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:28,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:28,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:28,800 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:27:31,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:31,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:31,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:31,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:31,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:36,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:38,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:39,364 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:27:41,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:41,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:41,843 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:41,843 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:41,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:43,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:43,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:43,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:27:45,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:45,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:47,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:51,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:51,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:51,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:51,849 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:52,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:52,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:52,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:58,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:58,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:58,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:58,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:00,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:01,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:01,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:01,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:01,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:03,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:07,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:28:07,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:09,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:09,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:28:12,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:12,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:12,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:12,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:14,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:28:14,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:14,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:14,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:14,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:19,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:21,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:22,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:22,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:22,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:22,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:25,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:27,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:27,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:27,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:27,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:28,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:28:28,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:29,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:29,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:29,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:30,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:35,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:35,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:39,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:28:40,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:44,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:44,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:44,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:46,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:51,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:56,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:59,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:59,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:59,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:01,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:06,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:09,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:29:11,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:14,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:14,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:14,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:17,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:21,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:21,999 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_5f4cc7e816317d29edbf.png with policy now +2023-07-12 06:29:22,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:22,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_a3ac4868653991969a01.png with policy now +2023-07-12 06:29:22,031 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:22,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_d7dea15105ec789e1d21.png with policy now +2023-07-12 06:29:22,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:22,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_cd95bda74fc7ee9c42fd.png with policy now +2023-07-12 06:29:22,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:22,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:29:22,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:29:22,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:29:22,339 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q6v3lzb6-media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 06:29:22,351 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mgjiihc2-media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 06:29:22,353 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lcbbsnbp-media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 06:29:22,362 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/opyq82er-media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 06:29:23,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:24,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:29:24,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 06:29:24,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 06:29:24,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 06:29:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 06:29:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:29:24,626 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jv1jesqd-media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 06:29:24,640 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mmui4izt-media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 06:29:24,650 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z0ubowlj-media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 06:29:24,683 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/75w12nqj-media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 06:29:28,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:29,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:29,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:29,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:31,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:31,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:33,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:38,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:39,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:39,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:29:41,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:41,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:29:41,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:29:41,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:29:44,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:44,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:44,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:44,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:45,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:29:45,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:50,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:50,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:50,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:29:50,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:29:50,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:29:52,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:29:52,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:55,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:59,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:59,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:59,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:59,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:00,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:01,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:01,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:01,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:01,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:06,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:06,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:06,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:09,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:30:11,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:12,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:12,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:12,124 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:12,126 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:13,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:13,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:14,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:30:14,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:30:14,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:16,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:20,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:22,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:22,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:22,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:22,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:22,965 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:27,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:27,552 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:27,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:30,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:30:30,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:30:30,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:32,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:33,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:33,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:33,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:33,545 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:34,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:34,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:38,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:39,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:30:41,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:43,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:43,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:43,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:43,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:43,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:45,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:30:45,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:30:45,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:48,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:48,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:48,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:53,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:53,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:53,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:53,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:53,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:55,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:55,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:58,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:58,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:58,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:58,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:59,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:00,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:00,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:00,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:02,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:31:02,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:31:04,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:09,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:31:10,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:15,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:15,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:15,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:15,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:20,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:25,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:30,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:30,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:30,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:31,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:36,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:39,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:31:42,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:45,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:45,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:45,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:47,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:52,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:53,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_011beff0267deaa0300b.png with policy now +2023-07-12 06:31:53,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_8efbafcfdf5d8df07263.png with policy now +2023-07-12 06:31:53,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_1b7f50867d7f974e4323.png with policy now +2023-07-12 06:31:53,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,520 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_bd10f02e7a1744e7844e.png with policy now +2023-07-12 06:31:53,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:31:53,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:31:53,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:31:53,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:31:53,797 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/217dm9zw-media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 06:31:53,801 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dlz8umik-media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 06:31:53,811 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7aug2hw-media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 06:31:53,833 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ytkm9a1t-media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 06:31:57,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:58,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:31:58,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:31:58,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 06:31:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 06:31:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 06:31:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 06:31:58,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:31:59,189 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tj914um9-media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 06:31:59,201 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b50vkfq7-media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 06:31:59,210 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cfjs8hhj-media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 06:31:59,212 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bt1whog2-media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 06:32:00,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:00,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:00,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:03,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:03,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:05,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:09,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:09,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:32:12,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:12,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:12,828 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:12,830 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:12,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:12,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:14,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:15,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:15,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:15,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:20,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:20,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:22,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:22,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:22,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:22,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:26,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:27,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:27,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:30,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:30,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:30,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:32,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:33,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:33,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:33,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:33,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:34,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:34,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:37,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:39,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:32:41,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:43,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:43,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:43,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:43,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:43,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:46,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:46,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:46,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:48,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:48,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:48,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:53,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:54,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:54,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:54,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:54,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:55,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:55,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:59,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:01,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:01,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:01,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:02,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:04,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:05,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:05,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:05,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:05,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:09,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:09,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:09,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:33:10,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:15,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:15,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:15,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:15,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:15,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:16,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:16,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:16,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:16,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:16,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:20,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:23,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:25,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:25,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:25,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:25,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:26,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:30,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:30,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:30,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:30,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:30,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:30,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:31,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:31,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:31,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:32,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:37,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:37,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:37,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:39,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:33:43,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:46,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:46,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:46,438 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:48,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:53,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:58,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:01,438 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:04,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:09,379 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:34:10,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:15,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:16,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:16,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:16,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:20,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:25,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_b236feda2a099cb72a65.png with policy now +2023-07-12 06:34:25,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,208 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_503b3c4a7c702e3bde04.png with policy now +2023-07-12 06:34:25,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_fb2e82a82bfa5264522f.png with policy now +2023-07-12 06:34:25,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_137a9e1a20d097ef6ead.png with policy now +2023-07-12 06:34:25,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:25,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:34:25,303 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:34:25,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:34:25,508 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tglkrqwx-media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 06:34:25,515 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/096qyqci-media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 06:34:25,571 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1uoeijv-media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 06:34:25,605 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cph02j2v-media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 06:34:26,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:26,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:34:26,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 06:34:26,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 06:34:26,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 06:34:26,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 06:34:26,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:34:26,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g0cffeqo-media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 06:34:26,858 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0jsdcsqm-media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 06:34:26,874 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zxto74ax-media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 06:34:26,892 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4p110s8g-media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 06:34:31,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:31,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:31,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:31,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:33,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:35,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:37,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:39,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:34:40,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:42,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:44,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:44,942 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:34:44,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:34:44,944 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:34:47,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:47,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:47,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:47,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:34:47,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:47,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:52,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:55,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:55,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:34:55,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:34:55,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:34:58,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:01,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:01,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:03,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:03,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:03,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:04,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:06,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:06,315 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:06,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:06,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:08,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:08,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:09,381 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:35:10,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:15,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:16,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:16,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:16,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:16,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:16,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:18,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:18,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:18,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:21,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:26,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:27,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:27,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:27,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:27,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:29,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:29,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:32,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:34,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:34,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:34,321 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:36,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:37,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:37,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:37,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:37,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:37,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:39,381 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:35:43,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:47,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:47,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:47,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:47,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:49,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:49,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:49,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:49,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:50,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:50,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:54,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:57,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:57,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:57,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:57,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:57,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:57,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:59,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:01,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:36:01,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:36:01,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:36:01,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:36:04,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:04,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:04,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:04,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:36:04,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:36:05,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:09,383 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:36:11,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:16,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:19,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:19,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:19,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:21,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:26,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:31,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:34,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:34,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:34,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:36,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:39,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:36:42,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:47,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:49,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:49,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:49,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:52,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:56,488 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,489 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_fb05536233b9d9384d99.png with policy now +2023-07-12 06:36:56,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_8b73b1d4908d57e0f731.png with policy now +2023-07-12 06:36:56,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_c996b3e0e81ceb767ce9.png with policy now +2023-07-12 06:36:56,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_cdd5f325de677a2f831c.png with policy now +2023-07-12 06:36:56,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:36:56,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:36:56,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:36:56,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:36:56,816 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9919zxeg-media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 06:36:56,823 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ykefeptl-media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 06:36:56,826 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ztuz0135-media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 06:36:56,831 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j6po7pvf-media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 06:36:57,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:01,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:01,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:01,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 06:37:01,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 06:37:01,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 06:37:01,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 06:37:01,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:37:01,413 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oanc6q8n-media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 06:37:01,423 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/psglyj6a-media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 06:37:01,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2593vaiy-media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 06:37:01,449 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1kkvcqv8-media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 06:37:03,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:04,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:04,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:04,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:06,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:08,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:08,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:09,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:37:14,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:15,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:16,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:16,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:16,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:16,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:20,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:20,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:20,137 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:20,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:22,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:22,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:25,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:25,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:25,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:25,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:25,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:29,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:29,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:31,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:35,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:35,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:35,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:36,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:36,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:36,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:36,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:36,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:36,190 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:39,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:37:42,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:43,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:43,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:46,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:46,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:46,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:46,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:47,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:50,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:50,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:50,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:50,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:50,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:52,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:55,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:55,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:55,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:55,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:57,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:57,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:57,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:02,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:04,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:05,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:38:05,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:05,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:05,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:05,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:05,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:05,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:08,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:09,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:38:11,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:38:11,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:13,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:15,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:15,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:15,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:15,464 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:18,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:38:18,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:19,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:20,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:38:20,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:20,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:24,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:25,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:26,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:26,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:26,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:26,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:30,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:31,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:31,440 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:31,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:31,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:32,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:38:32,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:36,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:36,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:38:36,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:36,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:39,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:38:39,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:41,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:46,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:51,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:51,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:52,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:57,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:02,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:06,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:06,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:06,325 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:07,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:09,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:39:12,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:17,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:21,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:21,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:21,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:23,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:25,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_934a30ce9258abe3af2d.png with policy now +2023-07-12 06:39:25,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_d7d4dbe10d69395e40ca.png with policy now +2023-07-12 06:39:25,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,673 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_37f9cd55db6f0502d41d.png with policy now +2023-07-12 06:39:25,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_cb784df1b13ebac5484a.png with policy now +2023-07-12 06:39:25,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:25,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:39:25,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:39:25,727 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:39:25,987 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgqjc1g2-media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 06:39:26,009 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ny3skwbi-media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 06:39:26,015 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/earbd7h5-media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 06:39:26,033 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wu6z9f2x-media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 06:39:28,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:28,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:39:28,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 06:39:28,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 06:39:28,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 06:39:28,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 06:39:28,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:39:29,010 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fnnp3zp1-media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 06:39:29,032 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xhvjah59-media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 06:39:29,032 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/os4jvlpk-media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 06:39:29,053 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g9uoglw8-media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 06:39:34,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:35,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:39:35,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:36,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:36,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:36,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:39,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:39:39,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:42,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:39:44,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:44,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:44,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:39:44,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:39:44,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:39:49,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:39:49,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:39:49,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:51,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:51,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:51,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:54,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:54,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:39:54,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:39:54,173 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:39:55,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:56,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:39:56,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:00,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:03,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:04,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:04,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:04,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:04,261 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:06,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:06,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:06,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:06,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:09,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:40:10,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:10,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:11,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:14,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:14,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:14,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:14,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:16,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:17,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:17,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:21,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:21,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:21,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:21,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:24,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:25,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:25,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:25,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:25,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:27,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:31,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:31,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:32,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:35,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:35,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:35,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:35,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:37,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:37,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:37,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:38,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:38,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:38,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:39,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:40:43,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:45,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:45,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:45,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:45,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:45,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:48,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:52,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:52,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:53,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:53,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:53,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:53,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:56,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:56,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:56,704 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:56,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:58,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:59,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:59,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:41:01,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:41:01,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:41:01,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:41:01,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:41:04,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:06,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:41:06,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:41:08,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:08,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:08,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:09,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:09,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:41:14,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:19,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:23,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:23,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:23,325 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:25,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:30,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:35,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:38,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:38,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:38,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:39,394 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:41:40,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:45,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:50,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:53,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:53,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:53,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:55,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:55,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:55,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_d7d78a7a562011ba37c0.png with policy now +2023-07-12 06:41:55,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:55,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_bada9f8f4e29cb81aa9e.png with policy now +2023-07-12 06:41:55,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:55,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_da4a4411337b18959a1d.png with policy now +2023-07-12 06:41:56,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:56,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_351faecc0b8489584c40.png with policy now +2023-07-12 06:41:56,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:41:56,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:41:56,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:41:56,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:41:56,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:41:56,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 06:41:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 06:41:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 06:41:56,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 06:41:56,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:41:56,276 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3gonoxnw-media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 06:41:56,304 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3l1qcaon-media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 06:41:56,309 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qgdltk7i-media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 06:41:56,395 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g21nu8lr-media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 06:41:56,794 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s4sujjm1-media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 06:41:56,912 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7k5lafm-media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 06:41:56,941 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l55i3t3e-media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 06:41:57,214 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8wd8s1c5-media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 06:42:01,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:03,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:06,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:06,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:08,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:08,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:08,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:09,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:42:10,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:11,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:16,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:16,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:16,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:16,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:16,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:17,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:17,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:22,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:23,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:23,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:23,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:24,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:26,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:26,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:26,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:26,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:27,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:31,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:31,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:33,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:36,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:36,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:36,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:36,639 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:38,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:38,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:38,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:39,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:39,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:39,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:39,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:42:44,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:45,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:47,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:47,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:47,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:47,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:49,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:52,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:52,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:54,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:54,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:54,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:55,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:57,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:57,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:57,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:57,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:59,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:59,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:00,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:06,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:06,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:07,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:07,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:07,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:07,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:09,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:43:09,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:09,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:09,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:11,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:13,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:13,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:17,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:17,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:17,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:17,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:17,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:20,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:20,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:23,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:24,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:24,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:24,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:26,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:26,767 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:26,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:26,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:27,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:27,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:28,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:31,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:31,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:31,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:31,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:34,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:34,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:34,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:39,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:39,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:43:39,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:39,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:39,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:44,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:49,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:54,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:54,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:54,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:55,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:00,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:05,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:09,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:44:09,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:09,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:09,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:10,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:15,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:20,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:24,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:24,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:24,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:25,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:26,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,517 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_a9454bcbee87eab5aea9.png with policy now +2023-07-12 06:44:26,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,526 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_9092d43f6e1c508c45c8.png with policy now +2023-07-12 06:44:26,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,536 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_214ff49c4cb987612b43.png with policy now +2023-07-12 06:44:26,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_aeea6b2da2c3bf58c7c3.png with policy now +2023-07-12 06:44:26,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:26,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:44:26,588 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:44:26,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:44:26,825 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ddiswf1u-media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 06:44:26,840 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vowqdifq-media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 06:44:26,857 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7dqhkmg-media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 06:44:26,891 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ym7mv0n-media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 06:44:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 06:44:30,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:44:31,059 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c9o3ytf5-media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 06:44:31,083 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kfg5k53t-media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 06:44:31,125 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/upcl89lk-media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 06:44:31,146 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dlgy8n3l-media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 06:44:31,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:36,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:36,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:37,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:44:39,402 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:44:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:39,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:41,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:44,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:44:46,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:46,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:46,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:44:46,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:44:46,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:44:51,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:44:51,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:44:51,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:54,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:54,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:54,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:56,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:56,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:44:56,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:44:56,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:44:57,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:58,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:44:58,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:02,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:05,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:06,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:06,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:06,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:06,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:07,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:09,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:45:09,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:09,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:09,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:12,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:12,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:12,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:17,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:17,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:17,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:17,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:18,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:19,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:19,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:23,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:25,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:25,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:25,441 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:26,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:28,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:28,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:28,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:28,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:29,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:33,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:34,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:39,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:39,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:39,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:39,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:39,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:45:40,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:40,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:40,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:41,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:41,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:41,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:46,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:48,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:49,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:49,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:49,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:49,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:51,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:55,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:55,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:56,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:56,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:56,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:57,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:58,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:58,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:58,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:58,616 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:46:02,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:46:02,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:46:03,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:03,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:46:03,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:46:03,200 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:46:03,202 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:46:08,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:09,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:46:09,137 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:46:09,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:46:11,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:11,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:11,348 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:14,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:19,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:24,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:26,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:26,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:26,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:29,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:34,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:39,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:46:40,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:41,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:41,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:41,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:45,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:50,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:55,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:56,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:56,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:56,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:57,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,838 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_1733cdf33e35398110c3.png with policy now +2023-07-12 06:46:57,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_b6d946329d7752479070.png with policy now +2023-07-12 06:46:57,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_f89b822e56af0bee16c1.png with policy now +2023-07-12 06:46:57,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_49a77c36b783c7c3533f.png with policy now +2023-07-12 06:46:57,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:46:57,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:46:57,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:46:57,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:46:58,171 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/afstmn9k-media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 06:46:58,186 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xl5pk6tt-media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 06:46:58,196 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/relcfh1z-media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 06:46:58,216 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mg9a4iul-media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 06:46:58,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:46:58,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 06:46:58,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 06:46:58,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 06:46:58,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 06:46:58,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:46:58,723 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cdkpx4pv-media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 06:46:58,738 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2ez19ru0-media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 06:46:58,755 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/at2ihzyc-media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 06:46:58,770 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fhvva6p6-media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 06:47:00,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:05,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:06,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:07,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:09,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:47:11,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:11,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:11,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:11,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:12,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:16,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:17,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:17,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:17,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:17,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:19,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:19,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:21,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:26,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:26,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:26,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:26,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:26,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:27,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:27,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:27,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:27,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:32,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:33,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:33,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:37,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:37,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:37,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:37,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:37,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:39,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:47:40,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:40,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:41,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:41,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:41,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:43,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:47,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:47,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:47,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:47,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:47,457 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:48,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:54,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:54,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:54,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:56,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:56,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:56,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:57,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:57,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:57,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:58,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:00,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:01,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:01,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:05,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:07,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:07,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:07,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:07,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:08,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:08,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:09,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:48:10,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:12,488 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:15,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:15,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:18,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:18,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:18,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:18,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:21,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:22,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:22,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:26,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:27,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:27,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:27,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:28,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:28,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:28,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:28,855 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:29,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:29,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:32,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:33,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:33,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:33,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:33,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:36,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:36,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:37,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:39,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:48:42,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:42,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:42,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:43,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:48,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:53,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:57,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:57,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:57,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:58,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:03,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:08,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:09,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:49:12,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:12,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:12,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:49:13,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:18,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:23,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:27,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_5f0c6d880a0a69fc8df8.png with policy now +2023-07-12 06:49:27,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_cab2ce99009b26517a5a.png with policy now +2023-07-12 06:49:27,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_5e34d10c6734761c103e.png with policy now +2023-07-12 06:49:27,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,526 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_0772971e314e9d07f59b.png with policy now +2023-07-12 06:49:27,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:27,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:49:27,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:49:27,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:49:27,849 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/db97lb2x-media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 06:49:27,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:27,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:27,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:49:27,874 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uz42jcku-media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 06:49:27,876 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b1l27awm-media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 06:49:27,879 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/67yfky8j-media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 06:49:29,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:32,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:49:32,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:32,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 06:49:32,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 06:49:32,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 06:49:32,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 06:49:32,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:49:33,537 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ii8z922h-media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 06:49:33,538 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rd8ky0z8-media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 06:49:33,589 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7p8hzmod-media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 06:49:33,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hzg29370-media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 06:49:34,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:37,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:39,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:49:39,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:40,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:43,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:43,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:43,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:49:45,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:46,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:46,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:49:46,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:49:46,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:49:46,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:49:46,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:51,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:53,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:56,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:56,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:56,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:49:56,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:49:56,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:49:58,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:58,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:58,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:00,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:00,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:02,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:06,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:06,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:06,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:06,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:07,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:07,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:08,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:09,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:50:13,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:13,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:14,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:16,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:16,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:16,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:16,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:18,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:22,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:22,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:23,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:27,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:27,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:27,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:27,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:28,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:28,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:28,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:29,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:29,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:29,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:35,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:36,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:38,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:38,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:38,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:38,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:39,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:50:40,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:43,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:43,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:43,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:43,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:43,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:45,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:48,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:48,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:48,562 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:48,564 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:50,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:50,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:50,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:56,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:57,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:58,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:58,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:58,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:58,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:58,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:58,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:58,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:01,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:02,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:51:02,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:51:02,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:51:02,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:51:04,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:51:04,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:51:07,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:09,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:51:11,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:51:13,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:13,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:13,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:13,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:18,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:23,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:28,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:28,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:28,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:28,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:33,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:38,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:39,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:51:43,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:43,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:43,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:44,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:49,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:54,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:57,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_72968a408eafe4615fe4.png with policy now +2023-07-12 06:51:57,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_de760783b75bdc640c99.png with policy now +2023-07-12 06:51:57,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_bcdd70acdf0c52445fdc.png with policy now +2023-07-12 06:51:57,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_661f798cc99bf53f5fc2.png with policy now +2023-07-12 06:51:57,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:51:57,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:51:57,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:51:57,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:51:57,797 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4exh7hav-media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 06:51:57,836 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/epiw2mfp-media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 06:51:57,868 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3300fubv-media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 06:51:57,872 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kju7u05q-media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 06:51:58,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:58,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:58,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:59,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:00,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:00,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 06:52:00,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 06:52:00,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 06:52:00,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 06:52:00,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:52:00,828 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ca3z86kw-media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 06:52:00,851 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3a41j9xi-media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 06:52:00,857 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fo4idme-media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 06:52:00,860 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r86dnv1g-media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 06:52:05,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:07,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:07,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:09,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:52:10,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:13,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:52:13,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:52:13,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:52:14,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:16,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:16,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:16,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:16,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:16,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:21,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:21,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:22,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:26,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:26,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:26,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:26,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:27,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:28,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:28,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:29,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:52:29,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:52:29,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:52:33,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:35,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:37,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:37,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:37,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:37,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:38,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:39,422 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:52:42,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:42,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:44,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:45,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:52:45,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:52:45,086 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:52:48,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:48,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:48,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:48,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:49,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:49,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:49,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:54,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:56,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:58,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:58,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:58,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:58,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:00,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:00,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:00,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:00,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:03,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:03,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:05,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:08,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:08,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:08,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:08,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:09,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:53:10,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:10,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:11,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:15,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:15,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:15,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:17,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:17,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:18,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:18,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:18,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:18,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:22,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:24,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:24,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:27,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:27,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:27,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:27,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:27,728 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:30,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:30,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:30,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:31,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:31,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:32,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:32,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:32,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:32,834 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:32,836 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:38,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:38,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:38,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:39,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:53:44,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:45,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:45,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:45,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:49,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:54,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:59,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:00,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:00,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:00,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:04,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:09,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:54:10,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:15,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:15,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:15,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:15,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:20,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:25,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:27,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_a26ecf3db23e2ccac681.png with policy now +2023-07-12 06:54:27,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_585881b9e0f6d40e0ec3.png with policy now +2023-07-12 06:54:27,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_68a5f846f45f697c2ead.png with policy now +2023-07-12 06:54:27,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_64d777e8b185c0851910.png with policy now +2023-07-12 06:54:27,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:27,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:54:27,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:54:27,611 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:54:27,848 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmaptz4j-media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 06:54:27,900 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5z5ilxw5-media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 06:54:27,903 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02974kkn-media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 06:54:27,909 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/twvu0oie-media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 06:54:28,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:54:28,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 06:54:28,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 06:54:28,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 06:54:28,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 06:54:28,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:54:28,375 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/63lurbvm-media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 06:54:28,396 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dt3p7hp1-media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 06:54:28,409 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0v0iy29p-media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 06:54:28,410 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dccb0g4a-media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 06:54:30,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:30,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:30,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:30,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:35,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:36,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:37,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:39,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:54:41,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:42,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:46,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:46,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:46,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:46,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:46,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:46,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:54:46,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:54:46,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:54:49,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:54:49,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:51,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:56,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:57,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:57,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:57,220 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:54:57,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:54:57,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:01,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:01,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:01,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:02,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:03,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:03,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:07,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:07,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:07,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:07,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:08,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:09,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:55:10,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:10,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:13,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:17,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:17,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:17,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:17,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:17,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:17,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:17,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:17,670 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:18,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:23,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:24,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:24,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:27,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:27,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:27,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:27,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:29,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:31,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:31,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:32,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:34,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:37,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:37,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:37,961 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:37,963 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:38,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:38,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:39,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:55:40,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:45,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:45,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:47,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:47,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:47,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:48,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:48,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:48,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:48,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:50,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:52,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:52,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:56,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:58,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:58,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:58,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:58,584 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:59,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:59,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:56:02,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:02,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:02,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:02,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:03,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:56:03,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:56:03,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:56:03,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:56:06,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:56:06,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:56:07,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:09,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:56:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:17,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:17,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:17,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:18,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:23,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:28,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:32,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:32,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:32,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:33,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:38,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:39,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:56:44,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:48,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:48,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:48,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:50,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:55,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:57,376 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_f24a9e2785b102661151.png with policy now +2023-07-12 06:56:57,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_dda90c1223ed981d3b1a.png with policy now +2023-07-12 06:56:57,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_c45fc62adfcc71af96fa.png with policy now +2023-07-12 06:56:57,406 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_3e49f286e7ccf53fddc7.png with policy now +2023-07-12 06:56:57,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:56:57,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:56:57,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:56:57,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:56:57,692 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qig2iph7-media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 06:56:57,695 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xkpfng0b-media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 06:56:57,724 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tvp3r7rj-media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 06:56:57,745 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tyrdholp-media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 06:57:00,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:02,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:02,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:02,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 06:57:02,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 06:57:02,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 06:57:02,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 06:57:02,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:57:02,978 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2vdvl5z-media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 06:57:02,988 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6who694h-media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 06:57:02,992 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/at3uw3m5-media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 06:57:02,997 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0gjv1zsf-media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 06:57:03,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:03,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:03,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:05,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:07,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:09,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:57:09,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:11,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:16,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:16,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:16,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:16,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:16,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:16,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:18,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:18,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:18,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:22,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:23,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:23,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:26,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:26,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:26,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:26,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:28,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:30,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:30,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:33,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:33,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:33,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:33,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:36,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:36,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:36,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:36,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:37,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:37,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:38,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:39,437 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:57:43,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:44,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:46,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:46,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:46,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:46,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:48,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:49,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:49,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:49,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:51,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:51,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:54,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:57,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:57,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:57,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:57,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:58,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:58,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:59,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:05,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:05,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:05,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:05,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:05,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:08,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:08,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:08,101 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:08,103 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:09,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:58:10,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:12,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:58:12,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:16,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:17,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:17,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:17,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:17,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:19,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:58:19,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:20,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:20,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:20,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:21,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:26,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:26,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:27,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:27,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:27,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:27,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:31,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:31,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:31,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:31,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:31,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:33,940 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:58:33,940 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:35,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:35,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:35,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:37,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:39,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:58:42,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:47,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:50,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:50,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:50,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:53,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:58,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:03,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:05,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:59:05,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:05,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:10,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:59:10,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:15,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:20,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:59:20,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:20,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:21,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:26,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_000cc75e10c8a7929ff7.png with policy now +2023-07-12 06:59:26,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_bcf133d130695d2171a4.png with policy now +2023-07-12 06:59:26,144 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,144 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_9e45cdde0ed95786481b.png with policy now +2023-07-12 06:59:26,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_d35268c957b3894ad085.png with policy now +2023-07-12 06:59:26,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:26,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:59:26,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:59:26,199 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:59:26,455 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h6r9t26o-media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 06:59:26,474 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uqifco14-media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 06:59:26,487 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gx027nuu-media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 06:59:26,671 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6j2t56r3-media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 06:59:27,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:30,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:59:30,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 06:59:30,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 06:59:30,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 06:59:30,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 06:59:30,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:59:30,593 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdfwa2wu-media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 06:59:30,593 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5d72aimf-media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 06:59:30,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xv41s7mo-media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 06:59:30,600 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xg6w7yke-media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 06:59:32,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:36,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:59:36,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:36,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:36,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:37,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:37,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:39,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:59:42,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:44,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:45,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:45,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:59:45,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:59:45,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:59:48,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:51,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:51,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:51,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:59:51,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:53,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:55,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:55,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:59:55,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:59:55,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:59:58,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:59:58,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:58,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:04,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:05,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:05,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:05,853 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:05,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:05,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:06,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:06,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:06,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:09,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:00:09,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:12,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:12,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:15,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:16,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:16,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:16,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:16,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:19,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:19,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:20,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:21,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:21,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:21,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:25,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:26,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:26,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:26,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:26,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:26,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:31,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:33,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:33,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:36,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:36,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:36,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:36,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:37,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:37,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:37,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:37,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:39,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:00:40,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:40,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:41,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:47,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:47,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:47,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:47,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:47,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:47,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:47,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:52,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:52,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:52,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:52,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:54,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:57,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:57,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:57,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:57,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:57,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:01:01,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:01:01,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:01:02,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:01:02,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:01:02,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:01:02,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:01:02,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:07,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:07,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:07,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:08,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:08,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:01:08,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:01:09,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:01:13,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:18,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:22,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:22,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:22,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:24,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:29,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:34,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:37,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:37,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:37,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:40,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:01:40,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:45,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:50,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:52,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:52,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:52,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:56,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:56,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_c0361568b66056400dc2.png with policy now +2023-07-12 07:01:56,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,971 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_f129804ddaa9a57cd45c.png with policy now +2023-07-12 07:01:56,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,982 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_4f3ac5bc86e6378f1a12.png with policy now +2023-07-12 07:01:56,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_f237ce678dbe9ec09de4.png with policy now +2023-07-12 07:01:57,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:01:57,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:01:57,034 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:01:57,036 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:01:57,346 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hupxmxau-media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 07:01:57,373 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/puhwr41s-media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 07:01:57,397 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v45tg3w1-media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 07:01:57,483 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yu8u3tvq-media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 07:01:57,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:01:57,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 07:01:57,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 07:01:57,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 07:01:57,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 07:01:57,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:01:58,154 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6oobafvm-media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 07:01:58,156 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5xqrq9q4-media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 07:01:58,188 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yrncacfi-media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 07:01:58,229 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0p7hjzcc-media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 07:02:01,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:04,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:06,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:06,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:07,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:07,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:07,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:09,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:02:11,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:12,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:15,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:15,868 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:15,868 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:15,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:17,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:18,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:18,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:22,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:22,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:22,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:23,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:25,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:25,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:25,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:25,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:25,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:28,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:32,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:32,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:33,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:36,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:36,263 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:36,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:36,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:38,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:38,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:38,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:39,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:02:39,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:39,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:39,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:46,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:47,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:47,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:47,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:47,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:50,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:53,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:53,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:54,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:54,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:54,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:55,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:57,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:57,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:57,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:57,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:00,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:00,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:01,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:06,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:07,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:07,456 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:07,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:07,458 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:08,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:08,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:09,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:03:09,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:03:09,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:09,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:12,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:15,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:17,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:17,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:17,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:17,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:17,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:22,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:22,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:23,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:24,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:24,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:03:24,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:28,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:28,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:28,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:28,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:28,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:29,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:33,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:33,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:33,275 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:33,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:33,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:36,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:36,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:38,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:39,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:03:39,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:39,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:44,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:49,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:54,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:54,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:54,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:59,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:04,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:09,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:04:09,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:09,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:10,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:15,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:20,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:24,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:04:24,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:24,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:25,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:27,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_731d1b60253f6f4063e4.png with policy now +2023-07-12 07:04:27,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,697 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_f040c34685a98d0f3850.png with policy now +2023-07-12 07:04:27,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_8d56acd02a2f045be997.png with policy now +2023-07-12 07:04:27,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_80603d23a803515f19ef.png with policy now +2023-07-12 07:04:27,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:27,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:04:27,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:04:27,763 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:04:28,047 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lvi4c5t4-media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 07:04:28,051 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x7chm46v-media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 07:04:28,064 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8fncz23m-media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 07:04:28,072 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjqx768t-media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 07:04:31,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:32,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:04:32,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:32,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 07:04:32,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 07:04:32,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 07:04:32,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 07:04:32,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:04:32,641 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jgu6c8s8-media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 07:04:32,646 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6m93moep-media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 07:04:32,670 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k5b8samo-media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 07:04:32,738 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mefxfa6f-media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 07:04:36,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:37,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:39,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:39,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:04:39,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:39,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:04:39,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:42,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:46,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:46,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:46,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:04:46,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:04:46,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:04:47,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:52,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:53,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:04:53,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:54,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:04:54,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:54,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:57,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:57,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:04:57,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:04:57,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:04:58,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:00,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:00,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:03,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:07,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:07,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:07,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:07,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:07,471 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:08,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:09,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:05:09,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:09,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:09,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:13,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:14,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:14,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:17,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:17,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:17,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:17,616 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:18,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:21,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:21,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:23,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:24,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:24,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:24,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:28,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:28,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:28,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:28,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:28,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:28,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:29,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:34,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:35,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:37,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:37,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:37,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:37,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:39,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:05:39,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:39,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:39,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:39,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:42,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:42,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:45,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:47,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:47,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:47,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:47,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:49,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:49,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:50,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:55,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:55,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:55,195 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:55,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:56,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:58,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:58,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:58,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:58,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:06:00,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:03,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:06:03,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:06:03,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:06:03,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:06:03,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:06:03,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:06:05,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:09,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:06:10,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:10,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:10,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:10,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:06:11,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:16,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:21,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:25,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:25,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:25,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:26,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:31,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:36,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:39,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:06:40,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:40,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:40,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:41,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:46,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:51,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:55,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:55,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:55,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:57,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:57,777 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png with policy now +2023-07-12 07:06:57,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_7295168e3d00a0e3d439.png with policy now +2023-07-12 07:06:57,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_fbcc4f86c9d83ede1f5d.png with policy now +2023-07-12 07:06:57,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,823 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_c1341754ea0507193be5.png with policy now +2023-07-12 07:06:57,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:06:57,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:06:57,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:06:57,850 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:06:58,122 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmausocq-media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 07:06:58,174 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q9k3rukw-media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 07:06:58,189 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z3poulg0-media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 07:06:58,190 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lptwt46n-media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 07:06:59,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:06:59,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 07:06:59,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 07:06:59,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 07:06:59,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 07:06:59,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:07:00,242 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v6116vk0-media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 07:07:00,253 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u1o3mubo-media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 07:07:00,269 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8dv0not5-media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 07:07:00,290 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b9a2wp2b-media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 07:07:03,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:06,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:08,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:08,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:09,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:07:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:07:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:10,866 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:13,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:13,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:18,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:18,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:18,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:18,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:18,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:20,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:20,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:24,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:25,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:25,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:27,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:28,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:28,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:28,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:28,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:29,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:34,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:34,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:35,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:38,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:38,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:38,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:38,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:39,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:07:40,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:40,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:40,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:41,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:41,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:45,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:48,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:48,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:48,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:48,517 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:49,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:49,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:50,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:55,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:55,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:55,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:56,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:57,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:58,001 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:58,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:58,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:01,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:03,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:03,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:07,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:08,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:08,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:08,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:08,270 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:09,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:08:10,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:10,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:10,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:08:12,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:17,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:17,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:19,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:19,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:19,143 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:19,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:22,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:24,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:24,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:26,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:08:26,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:26,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:08:28,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:29,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:29,907 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:29,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:29,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:31,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:31,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:33,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:34,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:34,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:34,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:34,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:38,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:38,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:39,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:39,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:08:41,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:08:41,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:41,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:08:44,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:49,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:54,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:56,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:08:56,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:56,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:00,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:05,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:09,466 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:09:11,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:11,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:11,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:11,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:16,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:21,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:26,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:26,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:26,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:26,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:28,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_f5c67f9540d8b5ed5c7c.png with policy now +2023-07-12 07:09:28,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_f31d3c5e28119395907e.png with policy now +2023-07-12 07:09:28,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,570 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_508c64e8a2ff4f92e95d.png with policy now +2023-07-12 07:09:28,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_46dc3e4359141341c6d4.png with policy now +2023-07-12 07:09:28,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:28,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:09:28,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:09:28,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:09:28,871 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kmyfeqhq-media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 07:09:28,883 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2wv7yiu4-media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 07:09:28,899 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fuw4ez18-media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 07:09:28,939 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c2ooy5h6-media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 07:09:32,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:34,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:09:34,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:34,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 07:09:34,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 07:09:34,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 07:09:34,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 07:09:34,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:09:34,854 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4bvzoqzb-media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 07:09:34,857 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0r170v3m-media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 07:09:34,859 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bew0rfft-media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 07:09:34,901 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/36byuuwt-media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 07:09:38,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:38,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:39,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:09:41,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:41,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:41,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:41,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:43,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:48,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:48,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:48,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:09:48,256 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:09:48,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:09:48,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:09:48,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:54,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:55,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:56,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:56,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:56,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:57,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:57,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:09:57,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:09:57,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:09:59,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:02,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:02,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:05,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:07,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:07,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:07,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:07,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:09,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:10:09,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:09,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:10,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:12,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:12,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:12,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:15,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:16,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:17,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:17,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:17,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:17,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:21,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:23,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:23,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:26,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:27,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:27,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:27,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:28,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:28,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:28,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:28,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:30,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:30,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:32,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:37,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:37,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:38,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:38,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:38,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:38,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:39,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:10:43,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:43,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:43,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:43,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:44,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:44,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:48,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:48,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:48,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:48,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:48,619 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:51,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:51,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:53,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:58,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:58,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:58,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:58,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:58,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:58,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:58,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:58,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:58,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:58,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:11:03,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:11:03,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:11:03,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:11:03,312 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:11:03,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:05,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:11:05,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:11:08,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:09,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:11:13,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:13,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:13,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:11:14,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:19,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:24,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:28,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:28,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:28,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:11:29,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:34,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:39,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:39,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:11:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:43,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:11:45,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:50,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:55,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:57,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,479 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_bbaae7ee7d09d30b3e1a.png with policy now +2023-07-12 07:11:57,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,489 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_92fe9627890fe0330f34.png with policy now +2023-07-12 07:11:57,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_be9cd4b95a81eb492abf.png with policy now +2023-07-12 07:11:57,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,512 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_603a92e14eb64a33c047.png with policy now +2023-07-12 07:11:57,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:11:57,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:11:57,560 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:11:57,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:11:57,800 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/trl7vgin-media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 07:11:57,807 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pojo9ykr-media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 07:11:57,812 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6c8obe6u-media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 07:11:57,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/osh8gvul-media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 07:11:58,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:58,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:58,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:00,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:02,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:02,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 07:12:02,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 07:12:02,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 07:12:02,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 07:12:02,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:12:02,430 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yisusads-media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 07:12:02,437 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/anevop3q-media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 07:12:02,487 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ddd126h-media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 07:12:02,496 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/41rkr40h-media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 07:12:05,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:07,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:09,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:09,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:12:11,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:13,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:13,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:16,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:16,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:17,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:17,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:17,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:17,521 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:21,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:23,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:23,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:26,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:27,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:27,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:27,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:27,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:28,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:28,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:30,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:30,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:32,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:37,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:37,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:37,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:37,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:37,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:37,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:39,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:12:42,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:43,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:44,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:44,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:47,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:47,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:47,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:47,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:47,810 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:51,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:51,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:53,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:58,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:58,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:12:58,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:58,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:58,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:58,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:58,300 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:58,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:59,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:05,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:05,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:05,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:08,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:08,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:08,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:08,805 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:09,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:13:10,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:12,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:12,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:13,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:13,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:13:13,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:15,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:18,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:18,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:18,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:18,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:19,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:19,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:21,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:26,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:26,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:28,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:28,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:28,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:28,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:28,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:28,499 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:32,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:33,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:33,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:33,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:33,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:33,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:33,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:37,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:39,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:13:40,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:43,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:13:43,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:43,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:43,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:48,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:53,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:58,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:58,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:13:58,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:58,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:03,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:08,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:09,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:14:13,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:13,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:13,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:14:14,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:19,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:24,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:27,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,820 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_80be20563f1620981e46.png with policy now +2023-07-12 07:14:27,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,830 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_2038713cb0876e29c6c8.png with policy now +2023-07-12 07:14:27,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_95d7597f938efa3e944e.png with policy now +2023-07-12 07:14:27,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_551525fb129586cd0c60.png with policy now +2023-07-12 07:14:27,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:27,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:14:27,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:14:27,895 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:14:28,116 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9rljlkhl-media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 07:14:28,141 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vd7hdrm4-media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 07:14:28,141 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ai28rglx-media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 07:14:28,171 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ew6jskpo-media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 07:14:28,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:28,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:28,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:14:29,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:14:29,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 07:14:29,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 07:14:29,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 07:14:29,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 07:14:29,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:14:30,018 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6mr8n795-media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 07:14:30,022 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xnpe6f38-media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 07:14:30,040 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wtg8jpjy-media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 07:14:30,110 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k0x370pl-media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 07:14:30,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:35,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:36,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:37,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:39,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:14:40,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:43,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:43,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:43,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:43,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:14:46,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:46,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:46,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:14:46,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:14:46,786 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:14:50,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:14:50,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:52,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:56,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:56,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:14:56,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:14:56,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:14:57,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:14:57,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:57,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:58,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:58,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:58,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:03,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:04,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:06,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:06,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:06,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:06,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:08,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:09,482 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:15:11,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:11,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:14,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:14,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:14,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:14,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:17,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:17,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:17,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:17,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:18,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:18,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:19,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:24,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:25,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:27,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:27,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:27,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:27,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:29,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:29,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:29,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:29,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:32,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:32,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:35,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:37,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:37,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:37,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:37,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:39,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:15:39,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:39,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:40,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:44,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:44,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:44,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:46,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:46,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:46,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:46,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:46,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:46,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:46,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:51,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:53,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:56,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:57,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:57,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:57,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:57,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:59,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:59,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:59,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:00,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:16:00,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:16:02,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:02,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:16:02,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:16:02,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:16:02,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:16:07,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:07,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:16:07,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:16:09,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:16:13,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:14,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:14,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:14,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:18,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:23,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:28,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:29,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:29,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:29,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:34,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:39,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:16:40,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:44,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:44,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:44,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:45,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:50,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:55,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:56,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_700fef9298b995bd049b.png with policy now +2023-07-12 07:16:56,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_9fb3356673dd42522db4.png with policy now +2023-07-12 07:16:56,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_a001a16142d0cdf3dd8e.png with policy now +2023-07-12 07:16:56,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_62bcdb68c12bb4eee832.png with policy now +2023-07-12 07:16:56,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:16:56,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:16:56,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:16:56,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:16:56,964 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zb40lzma-media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 07:16:56,972 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9rh15ulw-media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 07:16:56,978 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i1hxdi0u-media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 07:16:56,995 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2edkeqe2-media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 07:16:57,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:16:57,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 07:16:57,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 07:16:57,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 07:16:57,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 07:16:57,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:16:57,573 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/flm9oqff-media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 07:16:57,585 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4habb0yv-media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 07:16:57,586 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hs2yc8ck-media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 07:16:57,593 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m2ciyw1m-media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 07:16:59,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:59,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:59,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:00,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:04,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:06,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:06,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:09,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:17:11,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:11,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:14,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:17:14,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:17:14,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:16,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:16,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:16,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:16,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:16,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:18,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:18,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:22,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:25,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:25,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:25,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:25,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:25,961 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:27,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:30,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:17:30,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:17:30,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:32,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:32,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:33,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:35,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:35,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:35,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:35,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:38,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:39,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:39,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:39,487 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:17:44,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:17:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:17:45,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:46,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:46,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:46,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:46,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:46,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:46,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:50,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:53,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:55,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:56,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:56,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:56,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:56,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:00,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:00,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:00,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:00,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:00,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:00,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:06,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:06,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:06,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:06,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:06,191 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:09,488 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:18:12,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:14,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:15,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:15,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:15,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:16,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:16,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:16,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:16,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:17,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:21,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:21,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:22,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:26,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:26,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:26,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:26,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:28,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:28,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:30,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:30,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:30,875 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:31,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:31,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:31,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:31,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:33,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:35,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:35,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:38,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:39,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:18:43,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:45,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:45,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:45,875 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:48,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:53,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:58,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:00,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:00,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:00,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:04,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:09,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:09,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:19:14,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:15,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:15,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:15,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:20,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:25,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:25,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_4dc54de8f062b7449876.png with policy now +2023-07-12 07:19:25,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_9ead6b955150561d6d4c.png with policy now +2023-07-12 07:19:25,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_d91846de1cf74e316293.png with policy now +2023-07-12 07:19:25,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_bdda721ad2fd6569a964.png with policy now +2023-07-12 07:19:25,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:25,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:19:25,993 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:19:25,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:19:26,276 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/me3o9r19-media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 07:19:26,286 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sjybkwig-media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 07:19:26,293 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ri3lz2z-media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 07:19:26,323 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/arh639nj-media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 07:19:30,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:31,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:31,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:31,165 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:31,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:19:31,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 07:19:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 07:19:31,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 07:19:31,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 07:19:31,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:19:32,282 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/812paalf-media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 07:19:32,288 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jsd7m6an-media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 07:19:32,308 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yh8v3jgi-media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 07:19:32,324 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k67yy6wd-media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 07:19:35,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:36,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:38,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:39,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:19:42,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:44,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:44,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:19:44,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:19:44,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:19:46,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:46,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:46,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:46,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:19:46,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:48,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:53,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:54,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:54,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:19:54,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:19:54,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:19:54,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:19:54,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:59,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:02,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:02,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:02,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:02,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:04,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:05,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:05,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:05,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:05,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:09,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:20:10,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:10,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:10,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:15,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:16,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:16,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:16,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:16,444 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:17,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:17,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:17,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:18,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:18,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:21,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:26,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:26,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:26,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:26,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:26,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:26,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:26,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:32,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:32,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:32,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:32,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:35,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:36,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:36,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:36,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:36,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:37,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:39,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:20:43,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:43,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:43,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:47,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:47,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:47,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:47,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:47,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:47,328 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:47,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:49,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:51,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:51,112 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:54,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:57,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:57,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:57,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:57,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:59,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:59,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:59,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:02,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:21:02,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:21:02,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:21:02,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:21:02,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:02,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:02,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:04,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:07,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:21:07,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:21:09,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:21:10,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:15,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:17,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:17,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:17,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:21,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:26,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:31,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:32,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:32,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:32,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:36,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:39,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:21:41,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:46,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:47,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:47,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:47,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:52,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:56,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_1d177e30435b1c654b3d.png with policy now +2023-07-12 07:21:56,558 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,559 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png with policy now +2023-07-12 07:21:56,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_06dd5b28c3be7a9bbdb9.png with policy now +2023-07-12 07:21:56,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_94c6bfdf23b465d2a61f.png with policy now +2023-07-12 07:21:56,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:21:56,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:21:56,628 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:21:56,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:21:56,868 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/enpsc35o-media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 07:21:56,876 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e14us6j3-media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 07:21:56,880 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/egy1o4xm-media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 07:21:56,893 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xotiz8n9-media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 07:21:57,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:02,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:02,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:02,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:02,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:03,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:03,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:03,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 07:22:03,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 07:22:03,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 07:22:03,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 07:22:03,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:22:03,719 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wk6urojj-media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 07:22:03,725 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/23r2vrnb-media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 07:22:03,741 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x9ih07jo-media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 07:22:03,747 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wp4xrraw-media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 07:22:07,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:08,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:09,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:22:11,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:13,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:15,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:15,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:15,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:15,905 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:18,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:18,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:18,195 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:18,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:19,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:19,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:24,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:25,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:25,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:25,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:25,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:27,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:27,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:30,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:33,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:33,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:33,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:35,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:35,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:36,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:36,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:36,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:36,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:39,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:22:41,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:43,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:43,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:46,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:46,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:46,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:46,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:46,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:48,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:48,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:48,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:51,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:51,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:51,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:56,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:56,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:56,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:56,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:56,642 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:59,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:59,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:01,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:03,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:03,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:03,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:07,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:07,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:07,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:07,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:07,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:07,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:07,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:09,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:23:12,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:15,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:17,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:17,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:17,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:17,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:18,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:19,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:19,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:19,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:23,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:23,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:23,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:27,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:27,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:27,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:27,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:28,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:31,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:31,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:32,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:32,849 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:32,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:32,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:34,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:34,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:34,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:34,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:39,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:23:39,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:39,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:39,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:44,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:49,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:49,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:49,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:49,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:54,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:59,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:04,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:04,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:04,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:05,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:09,503 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:24:11,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:16,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:19,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:19,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:19,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:21,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:26,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:27,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_88b7298825f502e5a062.png with policy now +2023-07-12 07:24:27,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_0d6a954f28a281ccb3e6.png with policy now +2023-07-12 07:24:27,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,109 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png with policy now +2023-07-12 07:24:27,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_e4640ecdc8c22558e54b.png with policy now +2023-07-12 07:24:27,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:27,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:24:27,160 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:24:27,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:24:27,477 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpoyoq11-media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 07:24:27,480 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nvi9jur1-media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 07:24:27,482 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b6iclmr6-media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 07:24:27,521 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rnwpoach-media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 07:24:27,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:24:27,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 07:24:27,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 07:24:27,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 07:24:27,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 07:24:27,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:24:28,358 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tc6ejixr-media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 07:24:28,364 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/crbroj10-media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 07:24:28,365 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sva7a314-media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 07:24:28,369 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/urrbtt48-media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 07:24:32,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:34,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:34,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:34,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:35,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:24:37,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:37,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:39,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:24:43,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:43,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:24:46,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:46,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:24:46,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:24:46,958 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:24:48,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:49,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:49,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:49,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:51,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:24:51,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:24:54,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:56,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:56,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:24:56,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:24:56,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:24:59,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:24:59,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:00,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:05,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:25:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:05,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:06,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:07,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:07,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:07,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:07,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:08,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:08,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:09,505 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:25:11,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:16,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:16,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:17,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:17,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:17,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:17,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:19,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:20,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:22,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:24,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:24,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:27,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:27,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:27,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:27,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:27,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:32,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:32,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:33,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:34,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:35,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:37,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:37,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:37,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:37,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:38,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:39,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:25:40,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:40,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:44,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:48,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:48,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:48,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:48,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:48,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:50,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:25:50,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:50,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:50,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:55,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:56,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:56,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:57,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:57,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:57,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:57,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:26:01,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:02,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:26:02,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:26:02,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:26:02,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:26:04,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:26:04,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:26:05,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:05,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:05,348 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:07,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:09,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:26:12,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:17,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:20,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:20,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:20,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:23,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:28,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:33,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:35,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:35,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:35,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:38,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:39,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:26:44,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:49,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:50,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:50,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:50,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:54,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:56,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_c9724817961bea50e2eb.png with policy now +2023-07-12 07:26:56,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,881 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_f3ed24daee483f81fe21.png with policy now +2023-07-12 07:26:56,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_6f1689c0e9f7dc8526d4.png with policy now +2023-07-12 07:26:56,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_f03b061475b754e24cb5.png with policy now +2023-07-12 07:26:56,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:26:56,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:26:56,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:26:56,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:26:57,166 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m53zuuw2-media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 07:26:57,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/95blgi92-media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 07:26:57,227 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iwqbdmxa-media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 07:26:57,246 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/co7jg7gh-media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 07:26:59,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:00,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:00,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 07:27:00,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 07:27:00,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 07:27:00,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 07:27:00,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:27:00,817 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/31lp74ho-media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 07:27:00,852 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n7ik9cq0-media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 07:27:00,854 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2cz9xul7-media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 07:27:00,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yy9yqxyh-media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 07:27:04,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:05,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:27:05,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:05,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:06,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:08,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:09,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:27:10,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:15,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:15,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:15,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:15,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:16,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:16,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:16,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:20,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:27:20,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:20,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:21,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:24,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:25,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:25,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:25,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:25,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:27,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:32,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:32,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:32,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:35,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:35,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:35,490 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:35,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:35,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:35,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:37,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:39,511 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:27:40,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:40,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:42,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:45,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:45,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:45,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:45,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:48,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:48,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:48,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:50,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:50,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:53,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:56,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:56,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:56,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:56,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:56,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:56,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:58,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:04,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:04,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:06,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:06,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:06,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:06,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:06,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:06,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:06,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:09,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:28:09,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:12,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:12,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:15,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:17,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:17,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:17,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:17,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:20,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:20,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:20,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:21,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:21,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:21,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:26,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:28,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:28,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:28,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:28,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:28,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:28,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:31,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:32,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:32,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:32,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:33,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:36,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:36,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:36,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:36,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:36,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:37,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:39,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:28:42,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:47,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:51,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:51,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:51,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:53,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:58,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:03,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:07,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:07,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:07,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:08,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:09,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:29:13,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:18,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:22,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:22,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:22,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:24,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:27,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_e0be2e3dcde4725c11f7.png with policy now +2023-07-12 07:29:27,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_43277f4f939e843ab8c9.png with policy now +2023-07-12 07:29:27,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_437f85ccbcb8672d8508.png with policy now +2023-07-12 07:29:27,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_19d50aae0838de593b50.png with policy now +2023-07-12 07:29:27,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:27,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:29:27,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:29:27,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:29:27,552 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wgmt0qfb-media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 07:29:27,577 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yfz1hu8t-media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 07:29:27,595 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fzdgkf49-media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 07:29:27,626 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0u9b2c6t-media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 07:29:29,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:33,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:29:33,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:33,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 07:29:33,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 07:29:33,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 07:29:33,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 07:29:33,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:29:33,422 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fqm1r4d6-media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 07:29:33,435 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p3v4zms5-media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 07:29:33,474 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jkz1cilb-media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 07:29:34,314 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6tdvlxeq-media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 07:29:35,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:36,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:37,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:37,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:37,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:39,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:29:41,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:41,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:45,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:45,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:29:45,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:29:45,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:29:46,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:49,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:29:49,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:52,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:52,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:55,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:55,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:29:55,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:29:55,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:29:57,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:29:57,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:57,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:03,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:05,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:05,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:05,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:05,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:05,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:08,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:08,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:08,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:09,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:30:09,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:13,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:13,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:14,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:16,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:16,682 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:16,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:16,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:20,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:23,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:23,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:23,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:25,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:26,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:26,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:26,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:26,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:29,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:29,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:31,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:36,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:37,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:37,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:37,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:37,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:37,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:37,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:39,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:39,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:30:41,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:45,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:47,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:47,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:47,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:47,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:47,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:52,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:53,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:53,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:54,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:54,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:54,529 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:57,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:57,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:57,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:57,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:57,746 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:31:01,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:31:01,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:31:02,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:31:02,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:31:02,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:31:02,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:31:03,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:08,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:09,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:31:09,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:31:09,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:31:09,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:09,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:09,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:13,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:18,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:23,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:24,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:24,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:24,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:28,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:33,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:38,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:39,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:31:39,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:39,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:39,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:44,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:49,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:54,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:54,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:54,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:54,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:56,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:56,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_1fbb3840c9621213d4a8.png with policy now +2023-07-12 07:31:56,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:56,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_6c8d3f0f2e6689c4c999.png with policy now +2023-07-12 07:31:57,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:57,004 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_73398702cb61fca4ba23.png with policy now +2023-07-12 07:31:57,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:57,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_86559286380435a179bf.png with policy now +2023-07-12 07:31:57,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:31:57,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:31:57,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:31:57,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:31:57,299 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bnl9hdpd-media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 07:31:57,310 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vt37c6a2-media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 07:31:57,329 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e5wu5a9s-media/images/validation_1790_86559286380435a179bf.png +2023-07-12 07:31:57,353 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1ptuv03-media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 07:31:57,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:31:57,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 07:31:57,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 07:31:57,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_86559286380435a179bf.png +2023-07-12 07:31:57,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 07:31:57,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:31:58,058 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s0419ven-media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 07:31:58,088 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vy9v16li-media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 07:31:58,098 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8wmvpyud-media/images/validation_1790_86559286380435a179bf.png +2023-07-12 07:31:58,105 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d4lxbhr0-media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 07:32:00,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:05,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:05,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:07,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:09,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:32:09,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:09,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:09,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:10,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:13,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:16,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:16,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:16,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:16,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:16,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:21,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:21,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:22,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:25,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:25,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:25,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:26,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:26,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:26,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:26,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:28,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:29,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:29,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:33,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:36,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:36,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:36,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:36,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:37,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:37,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:38,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:39,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:32:40,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:40,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:40,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:44,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:45,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:46,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:46,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:46,698 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:46,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:49,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:53,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:53,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:54,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:56,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:56,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:56,603 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:57,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:57,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:57,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:57,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:59,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:01,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:01,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:05,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:07,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:07,980 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:07,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:07,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:09,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:33:09,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:09,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:10,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:12,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:12,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:12,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:15,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:17,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:18,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:18,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:18,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:18,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:20,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:25,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:25,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:26,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:27,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:27,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:27,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:28,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:28,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:28,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:28,043 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:32,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:32,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:32,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:32,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:32,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:33,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:33,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:38,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:39,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:33:41,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:42,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:42,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:42,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:44,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:49,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:54,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:57,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:57,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:57,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:59,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:04,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:09,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:34:10,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:12,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:12,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:12,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:16,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:21,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:26,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:27,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_1b5a6f042a0a4ba4a34a.png with policy now +2023-07-12 07:34:27,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_3003f5b283d5c2f3d3a3.png with policy now +2023-07-12 07:34:27,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_a5df1ab61da66e19d024.png with policy now +2023-07-12 07:34:27,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_c6b5b46f40b7e3cc06e5.png with policy now +2023-07-12 07:34:27,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:27,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:34:27,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:34:27,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:34:27,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:27,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:27,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:27,712 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lscsiwz4-media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 07:34:27,714 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wch5qjb7-media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 07:34:27,724 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q66ef5yj-media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 07:34:27,765 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mkrzwwhu-media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 07:34:30,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 07:34:30,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:34:30,556 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/til5tiea-media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 07:34:30,563 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qy6vriyq-media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 07:34:30,564 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e9vbjqij-media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 07:34:30,580 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fm9xo1m-media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 07:34:32,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:37,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:37,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:38,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:34:39,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:34:42,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:42,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:42,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:42,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:46,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:34:47,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:47,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:34:47,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:34:47,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:34:48,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:53,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:54,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:34:54,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:34:57,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:57,448 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:34:57,448 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:34:57,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:34:57,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:57,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:57,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:58,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:02,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:02,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:04,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:07,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:07,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:07,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:07,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:09,528 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:35:09,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:10,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:10,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:12,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:12,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:12,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:14,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:18,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:18,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:18,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:18,051 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:18,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:18,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:20,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:25,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:26,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:27,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:27,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:27,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:29,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:29,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:29,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:29,772 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:30,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:34,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:34,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:35,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:39,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:35:40,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:40,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:40,064 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:40,066 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:41,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:42,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:42,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:42,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:42,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:42,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:46,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:49,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:49,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:49,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:49,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:52,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:57,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:57,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:57,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:57,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:58,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:36:00,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:36:00,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:36:00,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:36:00,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:36:03,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:04,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:36:04,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:36:04,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:36:04,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:36:06,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:36:06,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:36:09,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:09,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:36:12,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:12,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:12,559 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:14,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:36:14,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:19,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:24,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:27,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:27,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:27,559 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:30,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:35,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:39,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:36:41,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:42,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:42,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:42,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:46,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:51,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:57,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:57,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:57,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:59,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_1d8ecdbc961ca85f19f9.png with policy now +2023-07-12 07:36:59,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_22b0716ab99e4f2c1baf.png with policy now +2023-07-12 07:36:59,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_db05126b5d482e1f86c1.png with policy now +2023-07-12 07:36:59,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_e9390a064f93495e881d.png with policy now +2023-07-12 07:36:59,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:36:59,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:36:59,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:36:59,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:36:59,650 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kpp2r0tp-media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 07:36:59,654 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t56qqajn-media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 07:36:59,702 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b6ex3ggg-media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 07:36:59,724 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8d4y3jcx-media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 07:37:02,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:02,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:02,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:02,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 07:37:02,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 07:37:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 07:37:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 07:37:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:37:03,187 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1qi9us5i-media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 07:37:03,198 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fw3pwp8l-media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 07:37:03,200 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9un616ma-media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 07:37:03,201 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/07g5gvxl-media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 07:37:07,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:09,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:09,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:37:10,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:13,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:13,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:13,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:13,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:18,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:18,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:18,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:18,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:18,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:18,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:23,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:26,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:26,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:28,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:28,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:28,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:29,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:29,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:29,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:29,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:29,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:34,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:34,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:34,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:39,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:37:39,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:39,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:39,860 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:39,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:39,862 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:42,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:42,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:44,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:44,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:44,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:44,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:50,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:50,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:50,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:50,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:50,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:50,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:50,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:56,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:58,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:59,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:59,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:59,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:59,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:59,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:59,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:59,763 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:01,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:06,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:07,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:07,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:09,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:38:10,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:10,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:10,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:10,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:12,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:14,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:14,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:14,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:38:15,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:15,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:17,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:21,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:21,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:21,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:21,849 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:22,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:23,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:23,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:27,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:29,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:29,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:29,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:38:31,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:31,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:31,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:31,868 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:31,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:33,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:36,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:36,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:36,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:36,603 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:39,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:39,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:39,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:39,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:38:44,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:44,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:44,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:38:44,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:49,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:54,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:59,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:59,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:59,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:00,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:05,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:09,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:39:10,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:14,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:14,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:14,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:16,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:21,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:26,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:29,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:29,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:29,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:30,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_0da500ef601e86506941.png with policy now +2023-07-12 07:39:30,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_7ecf7da22bc3b3fad59d.png with policy now +2023-07-12 07:39:30,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_3619de4ba8713b9bd91c.png with policy now +2023-07-12 07:39:30,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_953f6fe5b812a7050a46.png with policy now +2023-07-12 07:39:30,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:30,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:39:30,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:39:30,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:39:31,093 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ykyc3xt2-media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 07:39:31,098 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/77gmv7y5-media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 07:39:31,124 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gln7kaw4-media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 07:39:31,136 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jehgk0vs-media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 07:39:31,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:35,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:39:35,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:35,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 07:39:35,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 07:39:35,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 07:39:35,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 07:39:35,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:39:35,794 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s3pb70z6-media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 07:39:35,802 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bxw75m71-media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 07:39:35,808 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v9xl1yck-media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 07:39:35,809 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eolfm3fx-media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 07:39:37,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:39,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:39:40,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:43,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:43,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:44,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:44,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:44,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:48,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:49,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:49,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:39:49,483 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:39:49,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:39:51,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:39:51,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:53,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:59,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:59,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:59,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:59,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:59,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:59,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:59,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:39:59,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:39:59,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:04,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:09,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:09,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:40:10,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:10,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:10,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:10,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:14,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:40:14,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:40:14,800 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:40:14,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:15,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:15,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:20,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:20,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:20,270 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:20,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:20,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:23,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:23,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:25,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:30,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:30,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:40:30,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:40:30,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:40:31,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:31,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:31,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:31,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:31,554 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:35,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:39,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:40:39,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:39,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:41,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:42,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:42,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:42,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:42,200 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:45,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:40:45,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:40:47,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:47,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:47,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:52,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:52,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:52,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:52,328 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:52,330 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:55,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:55,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:57,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:02,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:02,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:02,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:02,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:02,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:41:02,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:41:02,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:41:02,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:41:03,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:41:03,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:41:06,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:41:06,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:41:06,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:41:06,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:41:08,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:09,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:41:11,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:41:11,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:41:14,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:17,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:17,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:17,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:20,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:25,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:30,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:32,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:32,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:32,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:35,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:39,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:41:40,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:45,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:47,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:47,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:47,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:51,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:56,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:01,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_7b892c3791e1a5536a80.png with policy now +2023-07-12 07:42:01,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_3cff7354bef942c8de30.png with policy now +2023-07-12 07:42:01,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_7b9c6d907af1a36abe9c.png with policy now +2023-07-12 07:42:01,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_d2e0fe71542ddbda5921.png with policy now +2023-07-12 07:42:01,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:01,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:01,292 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:01,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:01,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:01,550 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wp67pqbx-media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 07:42:01,551 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z8wys8t8-media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 07:42:01,565 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gvm63afe-media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 07:42:01,567 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2lm31y3d-media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 07:42:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:02,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:07,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:08,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:08,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:08,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 07:42:08,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 07:42:08,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 07:42:08,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 07:42:08,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:42:08,443 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rysdvin3-media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 07:42:08,452 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ps52ytx-media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 07:42:08,458 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e1sougx5-media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 07:42:08,466 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bqdevlby-media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 07:42:09,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:42:11,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:12,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:16,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:17,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:17,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:17,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:18,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:20,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:20,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:20,912 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:20,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:23,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:24,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:24,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:29,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:30,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:30,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:30,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:30,419 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:32,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:32,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:32,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:32,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:32,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:34,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:39,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:42:39,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:40,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:41,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:41,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:41,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:41,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:45,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:47,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:47,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:47,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:48,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:48,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:51,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:52,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:52,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:52,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:52,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:56,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:56,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:57,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:02,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:02,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:02,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:02,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:02,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:02,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:43:02,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:02,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:04,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:04,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:07,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:09,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:43:12,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:12,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:12,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:12,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:12,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:12,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:17,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:43:17,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:17,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:17,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:20,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:20,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:23,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:23,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:23,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:23,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:23,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:28,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:28,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:28,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:32,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:32,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:33,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:33,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:33,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:33,505 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:33,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:36,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:36,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:38,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:38,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:38,120 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:38,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:38,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:39,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:43:44,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:44,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:44,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:47,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:43:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:47,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:50,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:55,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:00,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:02,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:02,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:02,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:06,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:09,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:44:11,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:16,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:17,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:17,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:17,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:22,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:27,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:32,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:32,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_187a1a1f0c43344a0712.png with policy now +2023-07-12 07:44:32,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,539 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_31e70925ba342cb958b8.png with policy now +2023-07-12 07:44:32,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,549 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_2f7e6e14419630be9783.png with policy now +2023-07-12 07:44:32,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,559 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_b75b67c6b0d3f3cd134b.png with policy now +2023-07-12 07:44:32,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 07:44:32,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 07:44:32,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 07:44:32,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 07:44:32,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:44:32,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:44:32,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:44:32,600 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:44:32,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:44:32,859 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ztyw3cuk-media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 07:44:32,865 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k4lh9ym0-media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 07:44:32,897 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/irxmb2xs-media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 07:44:32,898 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6h2ws7hw-media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 07:44:32,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:32,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:32,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:33,299 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2o2jyzh9-media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 07:44:33,314 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xe429q4w-media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 07:44:33,713 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s5cp6p73-media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 07:44:33,751 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/54wz4pvo-media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 07:44:37,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:39,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:44:40,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:44:40,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:44:42,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:44:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:47,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:48,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:44:49,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:52,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:44:52,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:44:52,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:44:52,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:44:54,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:56,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:44:56,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:44:59,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:01,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:01,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:01,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:01,976 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:03,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:03,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:03,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:04,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:04,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:05,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:09,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:45:10,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:11,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:11,637 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:11,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:11,639 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:12,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:12,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:15,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:19,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:19,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:19,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:20,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:20,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:22,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:22,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:22,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:22,232 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:25,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:28,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:28,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:31,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:31,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:31,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:31,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:31,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:34,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:34,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:34,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:36,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:36,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:36,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:39,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:45:41,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:41,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:41,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:41,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:42,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:44,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:44,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:48,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:49,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:49,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:49,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:52,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:52,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:52,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:52,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:52,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:52,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:53,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:58,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:00,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:46:03,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:46:03,422 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:46:03,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:46:03,424 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:46:04,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:05,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:46:05,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:05,226 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:07,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:46:07,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:46:07,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:46:07,958 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:46:08,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:46:08,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:46:09,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:46:09,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:14,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:16,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:46:19,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:20,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:20,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:46:20,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:25,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:30,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:35,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:35,226 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:35,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:39,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:46:40,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:45,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:50,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:46:50,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:50,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:51,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:56,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:01,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:02,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_82f6b3c68a48fade34f5.png with policy now +2023-07-12 07:47:02,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_3a3b446f556528dde605.png with policy now +2023-07-12 07:47:02,288 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_ec640dd671161721e4f7.png with policy now +2023-07-12 07:47:02,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,298 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_c776faecdec7dde7f595.png with policy now +2023-07-12 07:47:02,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:02,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:02,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:02,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:02,595 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0h30y3m3-media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 07:47:02,595 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cuv84jfo-media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 07:47:02,613 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ea6r8vdc-media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 07:47:02,635 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/12jqscka-media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 07:47:05,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:05,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 07:47:05,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 07:47:05,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 07:47:05,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 07:47:05,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:47:05,506 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6fu3y29p-media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 07:47:05,529 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gfuoc84r-media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 07:47:05,533 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y7va5w5v-media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 07:47:05,554 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bote0wz2-media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 07:47:05,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:05,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:05,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:06,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:09,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:47:11,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:12,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:13,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:17,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:21,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:21,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:21,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:21,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:22,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:22,321 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:22,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:22,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:23,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:28,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:29,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:29,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:31,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:31,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:31,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:31,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:34,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:36,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:36,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:36,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:37,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:37,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:39,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:39,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:47:41,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:41,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:41,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:41,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:45,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:45,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:45,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:50,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:51,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:51,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:51,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:52,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:52,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:52,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:52,405 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:53,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:53,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:56,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:01,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:01,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:03,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:03,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:03,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:03,270 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:06,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:06,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:06,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:06,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:09,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:09,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:09,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:48:11,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:14,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:14,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:14,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:14,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:17,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:17,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:17,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:21,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:21,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:21,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:22,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:24,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:24,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:24,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:24,196 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:25,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:25,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:28,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:33,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:33,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:34,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:34,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:34,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:34,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:36,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:36,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:36,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:39,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:39,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:39,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:39,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:39,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:39,558 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:48:41,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:41,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:44,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:49,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:49,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:51,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:51,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:51,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:59,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:04,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:06,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:06,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:06,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:09,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:49:10,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:15,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:20,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:21,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:21,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:21,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:25,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:30,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:33,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_7b660814a00507637354.png with policy now +2023-07-12 07:49:33,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_2bee3f76993cc3a6ba8f.png with policy now +2023-07-12 07:49:33,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_2ec1726523e7df6ef87a.png with policy now +2023-07-12 07:49:33,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_763a0499812940f346e6.png with policy now +2023-07-12 07:49:33,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:49:33,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:49:33,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:49:33,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:49:33,460 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/glfcdekb-media/images/validation_1860_7b660814a00507637354.png +2023-07-12 07:49:33,482 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6968sm3s-media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 07:49:33,503 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/erdw3tvn-media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 07:49:33,504 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dxb4e7uq-media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 07:49:36,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:37,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:37,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:37,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:37,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:49:37,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 07:49:37,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_7b660814a00507637354.png +2023-07-12 07:49:37,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 07:49:37,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 07:49:37,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:49:38,125 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vw28brbv-media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 07:49:38,134 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/17bhwyml-media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 07:49:38,134 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vuor62zl-media/images/validation_1860_7b660814a00507637354.png +2023-07-12 07:49:38,186 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9kxak9dk-media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 07:49:39,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:49:42,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:42,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:49:45,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:49:48,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:51,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:49:51,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:49:51,535 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:49:51,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:49:52,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:52,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:52,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:53,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:53,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:49:53,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:49:59,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:01,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:01,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:01,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:01,650 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:01,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:01,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:04,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:07,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:07,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:07,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:09,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:50:09,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:09,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:11,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:11,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:11,428 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:11,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:14,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:17,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:17,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:20,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:22,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:22,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:22,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:22,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:22,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:22,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:22,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:25,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:30,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:32,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:32,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:32,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:32,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:33,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:33,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:36,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:37,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:37,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:37,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:39,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:50:41,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:42,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:43,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:43,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:43,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:43,238 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:47,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:50,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:50,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:52,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:52,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:52,834 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:52,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:53,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:53,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:53,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:53,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:57,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:58,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:58,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:51:03,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:04,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:51:04,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:51:04,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:51:04,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:51:06,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:51:06,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:51:08,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:08,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:08,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:08,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:51:08,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:51:08,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:51:08,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:51:09,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:09,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:51:14,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:51:14,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:51:14,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:19,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:23,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:23,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:23,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:25,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:30,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:35,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:38,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:38,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:38,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:39,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:51:40,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:45,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:50,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:53,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:53,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:53,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:56,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:01,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:02,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_c031f9ce29973bad20f5.png with policy now +2023-07-12 07:52:02,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,529 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_386744990d75b0d3adfe.png with policy now +2023-07-12 07:52:02,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_96924703d0889517e752.png with policy now +2023-07-12 07:52:02,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,531 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_b0e70d9482eb144ce2cd.png with policy now +2023-07-12 07:52:02,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:02,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:02,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:02,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:02,830 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/52h52k29-media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 07:52:02,874 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ap2wy70a-media/images/validation_1870_96924703d0889517e752.png +2023-07-12 07:52:02,876 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hcn5wvtl-media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 07:52:02,884 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kdup5iap-media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 07:52:06,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:08,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:52:08,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:08,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:09,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:52:10,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 07:52:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_96924703d0889517e752.png +2023-07-12 07:52:10,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 07:52:10,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 07:52:10,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:52:10,700 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cpo472zj-media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 07:52:10,706 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dq263vvf-media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 07:52:10,731 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nyz7hxep-media/images/validation_1870_96924703d0889517e752.png +2023-07-12 07:52:10,731 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wh8s8xxp-media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 07:52:12,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:12,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:17,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:18,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:21,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:21,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:21,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:21,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:23,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:52:23,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:23,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:23,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:26,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:26,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:28,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:31,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:31,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:31,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:31,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:33,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:34,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:34,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:38,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:39,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:52:39,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:39,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:39,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:52:42,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:42,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:42,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:42,114 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:42,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:42,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:44,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:49,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:50,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:53,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:53,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:53,717 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:53,719 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:54,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:54,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:55,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:58,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:58,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:00,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:03,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:03,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:03,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:03,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:05,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:06,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:06,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:09,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:09,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:09,568 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:53:11,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:13,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:13,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:13,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:13,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:14,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:14,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:17,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:22,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:22,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:23,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:23,981 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:23,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:23,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:24,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:24,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:27,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:30,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:30,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:32,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:34,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:34,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:34,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:34,402 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:38,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:38,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:38,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:38,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:38,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:38,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:38,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:39,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:39,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:39,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:53:44,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:46,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:46,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:54,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:53:54,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:54,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:55,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:00,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:05,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:09,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:09,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:09,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:09,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:54:10,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:15,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:20,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:24,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:24,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:24,357 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:26,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:31,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:33,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_cf1d81e3b206f0eeddf8.png with policy now +2023-07-12 07:54:33,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_e42387963e6467795ac3.png with policy now +2023-07-12 07:54:33,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,052 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_a24606b4de934e6b61c1.png with policy now +2023-07-12 07:54:33,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_19c60f744dd634323456.png with policy now +2023-07-12 07:54:33,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:54:33,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:54:33,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:54:33,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:54:33,381 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/57xkdxgj-media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 07:54:33,394 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g27zamfh-media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 07:54:33,395 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vgvbgnoe-media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 07:54:33,577 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nw35bl6y-media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 07:54:34,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:54:34,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 07:54:34,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 07:54:34,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 07:54:34,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 07:54:34,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:54:35,312 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zdaihrrq-media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 07:54:35,362 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0nkvj53q-media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 07:54:35,367 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4b5ldon-media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 07:54:35,402 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6tgh2zir-media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 07:54:36,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:39,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:39,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:39,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:39,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:54:41,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:42,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:54:44,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:54:46,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:50,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:54:52,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:54,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:54:54,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:54:54,029 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:54:54,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:54:54,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:54,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:54,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:58,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:58,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:54:58,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:03,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:03,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:03,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:03,741 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:03,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:06,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:06,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:09,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:09,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:55:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:10,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:13,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:13,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:13,244 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:13,246 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:15,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:20,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:23,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:24,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:24,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:24,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:24,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:25,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:25,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:25,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:26,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:31,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:31,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:31,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:34,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:34,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:34,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:34,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:37,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:39,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:39,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:39,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:55:40,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:40,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:40,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:43,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:45,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:45,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:45,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:45,015 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:47,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:47,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:48,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:54,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:55,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:55,075 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:55,075 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:55,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:55,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:55,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:55,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:55,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:55,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:00,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:03,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:04,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:56:04,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:56:04,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:56:04,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:56:05,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:09,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:56:09,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:56:09,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:56:09,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:56:09,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:56:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:10,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:11,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:56:11,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:11,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:16,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:19,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:21,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:25,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:25,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:25,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:27,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:32,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:37,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:39,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:56:40,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:40,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:40,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:42,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:47,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:52,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:55,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:58,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:03,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:04,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_5fabc2eb5b38fc6c441f.png with policy now +2023-07-12 07:57:04,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,267 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_362eab872b8e6854bfcb.png with policy now +2023-07-12 07:57:04,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_abe953372839c2e35d11.png with policy now +2023-07-12 07:57:04,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_61843d74c0b1b4039c5a.png with policy now +2023-07-12 07:57:04,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:04,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:04,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:04,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:04,550 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vs54hd9o-media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 07:57:04,557 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gx3sul2d-media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 07:57:04,574 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w6rqmfhn-media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 07:57:04,574 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8057oeb7-media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 07:57:07,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:07,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 07:57:07,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 07:57:07,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 07:57:07,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 07:57:07,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:57:07,859 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/odann1ub-media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 07:57:07,860 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9vpluri5-media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 07:57:07,867 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2wol7bz2-media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 07:57:07,871 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i7mxby7b-media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 07:57:09,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:09,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:57:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:10,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:14,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:14,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:15,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:20,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:23,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:23,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:23,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:23,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:23,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:25,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:25,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:25,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:25,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:30,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:31,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:31,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:33,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:33,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:33,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:33,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:35,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:39,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:39,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:39,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:57:40,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:40,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:40,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:40,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:43,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:43,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:43,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:43,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:46,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:47,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:47,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:51,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:54,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:54,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:54,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:54,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:55,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:55,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:55,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:55,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:55,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:57,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:02,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:03,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:05,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:05,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:05,180 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:05,182 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:08,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:09,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:58:10,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:10,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:10,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:11,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:11,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:14,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:14,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:14,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:14,978 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:14,980 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:19,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:19,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:19,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:25,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:25,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:25,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:25,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:25,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:25,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:25,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:25,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:27,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:27,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:31,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:35,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:35,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:35,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:35,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:35,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:35,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:37,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:39,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:58:39,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:39,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:39,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:39,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:40,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:40,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:40,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:42,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:43,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:43,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:47,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:52,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:55,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:55,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:55,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:57,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:02,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:07,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:09,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:59:10,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:10,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:10,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:12,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:17,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:22,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:25,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:25,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:25,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:27,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:32,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:34,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,311 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_755c75927be8fb0bccfb.png with policy now +2023-07-12 07:59:34,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_7ac136e8ccfb72e61b40.png with policy now +2023-07-12 07:59:34,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,331 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_e8722f8bf9b351a33b0d.png with policy now +2023-07-12 07:59:34,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_1895be2b314f5c3cc922.png with policy now +2023-07-12 07:59:34,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:59:34,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:59:34,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:59:34,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:59:34,659 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qa1zdyls-media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 07:59:34,661 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpp3wrnq-media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 07:59:34,674 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h36y1t5f-media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 07:59:34,693 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e9kncinm-media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 07:59:38,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:39,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:59:40,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:59:40,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:59:40,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 07:59:40,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 07:59:40,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 07:59:40,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 07:59:40,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:59:40,540 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zvb8vy5h-media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 07:59:40,546 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fkm40qm2-media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 07:59:40,561 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9zmd740-media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 07:59:40,580 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fku2fgn6-media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 07:59:40,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:40,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:40,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:43,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:43,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:59:48,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:59:49,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:53,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:59:53,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:59:53,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:59:53,164 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:59:55,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:55,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:55,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:55,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:56,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:59:56,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:00,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:02,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:02,564 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:02,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:02,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:04,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:04,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:06,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:09,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:00:10,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:00:10,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:10,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:11,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:12,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:12,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:12,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:12,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:12,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:17,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:20,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:20,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:23,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:23,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:23,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:23,445 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:23,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:25,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:00:25,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:25,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:28,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:28,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:29,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:33,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:33,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:33,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:33,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:34,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:36,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:36,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:39,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:00:40,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:40,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:40,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:43,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:43,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:43,620 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:43,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:44,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:44,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:45,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:50,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:52,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:54,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:55,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:55,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:55,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:55,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:55,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:56,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:00,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:01:00,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:01:02,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:05,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:01:05,425 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:01:05,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:01:05,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:01:07,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:08,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:01:08,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:01:09,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:01:10,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:01:10,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:01:10,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:01:10,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:01:10,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:10,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:12,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:16,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:01:16,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:01:17,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:22,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:25,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:01:25,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:25,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:27,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:32,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:37,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:39,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:01:40,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:01:40,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:40,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:43,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:48,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:53,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:55,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:01:55,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:55,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:59,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:04,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_6fd922b416ffd009954a.png with policy now +2023-07-12 08:02:04,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_5760a804d5e3912bf871.png with policy now +2023-07-12 08:02:04,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,733 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_34349e41d8bc032e1c7d.png with policy now +2023-07-12 08:02:04,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_f7431e028249d68b3d9b.png with policy now +2023-07-12 08:02:04,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:04,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:04,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:04,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:05,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:07,696 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6r27yrl0-media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 08:02:07,815 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wdmtgtar-media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 08:02:09,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:02:10,529 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zrpuc7z1-media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 08:02:10,661 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rbsqecva-media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 08:02:10,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:10,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:10,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:10,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 08:02:12,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 08:02:12,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 08:02:12,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:02:13,075 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/giwccv5d-media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 08:02:13,077 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c1mhn3g8-media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 08:02:13,083 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vhzs1dx5-media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 08:02:13,105 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/76gvkkqt-media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 08:02:15,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:16,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:20,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:22,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:25,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:25,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:25,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:25,523 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:25,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:25,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:25,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:27,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:28,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:28,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:33,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:34,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:34,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:34,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:34,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:36,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:36,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:38,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:39,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:02:40,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:40,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:40,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:44,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:44,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:45,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:45,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:45,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:45,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:49,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:52,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:52,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:55,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:55,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:55,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:55,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:56,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:56,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:56,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:56,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:00,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:00,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:00,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:05,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:05,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:05,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:05,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:05,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:08,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:08,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:09,591 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:03:10,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:10,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:10,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:10,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:16,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:16,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:16,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:16,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:16,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:17,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:17,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:21,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:25,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:26,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:26,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:27,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:27,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:27,160 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:27,162 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:31,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:33,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:33,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:37,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:37,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:37,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:37,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:37,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:39,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:03:41,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:41,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:41,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:41,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:41,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:41,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:41,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:42,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:47,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:49,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:49,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:52,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:56,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:56,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:57,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:02,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:07,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:09,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:04:11,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:11,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:11,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:13,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:18,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:23,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:26,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:26,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:28,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:33,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:36,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_0c56e18802b80a5e1cdb.png with policy now +2023-07-12 08:04:36,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_7d921cd7daa3b8f8aae7.png with policy now +2023-07-12 08:04:36,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,353 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_60d58608f7bfc9ddde76.png with policy now +2023-07-12 08:04:36,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_3e7c1f6543b9b9815c1e.png with policy now +2023-07-12 08:04:36,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:04:36,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:04:36,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:04:36,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:04:36,809 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j8e2dk62-media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 08:04:36,813 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r057h9bw-media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 08:04:36,817 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wac6l1b5-media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 08:04:36,818 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hltbak8g-media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 08:04:37,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:04:37,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 08:04:37,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 08:04:37,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 08:04:37,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 08:04:37,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:04:37,781 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9jmiw89h-media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 08:04:37,800 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gezev9am-media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 08:04:37,810 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cozq7609-media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 08:04:37,814 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/494own9a-media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 08:04:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:39,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:04:41,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:41,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:41,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:44,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:45,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:04:46,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:04:50,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:53,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:04:56,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:56,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:57,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:04:57,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:04:57,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:04:57,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:01,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:01,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:01,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:06,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:06,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:06,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:06,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:06,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:09,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:09,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:09,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:05:11,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:11,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:11,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:11,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:16,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:16,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:16,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:16,416 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:17,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:17,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:17,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:23,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:25,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:26,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:26,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:26,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:26,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:26,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:26,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:26,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:28,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:33,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:33,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:34,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:36,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:36,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:36,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:36,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:39,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:39,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:05:41,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:41,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:41,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:41,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:41,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:44,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:47,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:47,381 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:47,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:47,383 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:49,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:49,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:49,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:55,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:56,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:56,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:56,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:57,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:57,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:57,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:57,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:57,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:06:00,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:05,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:06:05,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:06:06,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:08,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:06:08,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:06:08,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:06:08,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:06:09,596 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:06:11,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:11,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:11,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:11,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:13,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:06:13,482 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:06:13,483 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:06:13,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:06:13,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:06:13,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:06:17,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:21,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:06:22,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:26,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:26,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:26,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:27,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:32,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:37,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:39,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:06:41,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:41,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:41,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:42,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:47,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:52,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:56,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:56,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:56,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:58,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:03,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:07,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_48056833d0b2d7f2157c.png with policy now +2023-07-12 08:07:07,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_dc6aa83bfa1de0638963.png with policy now +2023-07-12 08:07:07,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_8bf913d97ecc12b8092b.png with policy now +2023-07-12 08:07:07,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,751 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_cbdfa1d5c7889c908498.png with policy now +2023-07-12 08:07:07,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:07,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:07,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:07,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:08,070 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5p2jy24k-media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 08:07:08,076 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/syjjzg8r-media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 08:07:08,100 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8i4sb78p-media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 08:07:08,178 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rxm04c7j-media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 08:07:09,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:07:09,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:09,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 08:07:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 08:07:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 08:07:09,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 08:07:09,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:07:10,397 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g63bbsed-media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 08:07:10,495 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p46ibllb-media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 08:07:10,518 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n46ri2nu-media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 08:07:10,568 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/65gh3t8l-media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 08:07:12,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:12,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:12,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:14,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:17,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:17,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:19,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:25,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:25,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:27,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:27,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:27,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:27,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:27,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:27,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:27,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:30,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:33,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:33,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:35,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:37,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:37,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:37,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:37,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:39,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:07:41,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:42,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:42,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:42,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:42,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:42,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:46,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:48,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:48,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:48,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:48,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:50,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:50,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:52,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:57,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:58,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:58,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:58,026 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:58,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:58,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:58,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:58,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:58,592 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:03,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:06,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:06,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:08,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:08,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:08,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:08,923 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:08,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:09,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:08:13,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:13,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:13,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:08:14,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:14,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:14,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:19,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:19,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:19,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:19,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:19,240 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:22,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:22,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:24,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:28,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:28,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:28,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:08:28,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:28,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:28,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:28,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:29,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:30,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:30,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:34,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:38,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:38,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:38,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:38,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:38,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:39,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:08:40,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:43,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:43,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:43,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:43,395 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:43,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:43,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:43,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:08:46,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:46,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:46,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:51,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:56,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:58,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:58,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:58,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:01,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:06,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:09,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:09:12,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:13,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:13,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:13,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:17,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:22,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:27,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:28,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:28,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:28,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:32,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:37,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:38,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,139 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_55fed6437ec58a4992ec.png with policy now +2023-07-12 08:09:38,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_b7870fe1d164423912ca.png with policy now +2023-07-12 08:09:38,158 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_36d3d88a96d60bb7d69e.png with policy now +2023-07-12 08:09:38,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_42b89bc7a5b6ad127dd2.png with policy now +2023-07-12 08:09:38,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:09:38,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:09:38,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:09:38,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:09:38,565 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lkpgn2yi-media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 08:09:38,610 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9z4722yk-media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 08:09:38,614 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/26hbi3v9-media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 08:09:38,645 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1j3rzjoe-media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 08:09:39,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:09:42,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:09:42,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:09:42,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 08:09:42,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 08:09:42,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 08:09:42,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 08:09:42,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:09:42,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:43,065 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bvznvbhy-media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 08:09:43,078 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kfleqsdc-media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 08:09:43,087 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vinz16m4-media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 08:09:43,211 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u7lxxjj8-media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 08:09:43,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:43,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:43,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:47,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:09:48,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:50,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:09:54,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:56,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:09:56,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:09:56,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:09:56,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:09:58,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:09:58,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:09:58,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:58,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:58,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:59,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:05,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:06,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:06,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:06,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:06,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:06,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:06,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:09,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:10:11,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:13,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:13,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:13,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:14,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:16,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:16,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:16,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:16,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:16,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:21,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:22,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:22,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:26,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:27,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:27,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:27,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:27,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:29,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:29,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:29,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:30,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:30,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:31,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:37,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:37,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:37,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:37,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:37,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:38,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:38,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:39,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:10:42,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:44,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:44,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:44,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:46,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:48,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:49,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:49,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:49,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:49,079 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:53,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:54,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:54,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:58,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:58,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:58,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:58,602 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:59,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:59,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:59,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:59,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:02,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:11:02,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:11:05,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:09,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:11:09,127 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:11:09,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:11:09,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:11:09,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:11:10,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:11,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:11:11,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:11:13,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:11:13,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:11:13,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:11:13,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:11:14,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:14,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:14,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:11:16,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:19,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:11:19,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:11:21,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:26,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:29,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:29,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:29,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:11:32,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:37,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:39,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:11:42,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:44,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:44,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:44,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:11:48,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:53,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:58,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:59,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:59,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:59,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:03,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:08,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_2c29e26c44ad93206dd5.png with policy now +2023-07-12 08:12:08,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_a1d43d5ef09db63ec9f2.png with policy now +2023-07-12 08:12:08,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_a1baa48379524c1c8b17.png with policy now +2023-07-12 08:12:08,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_e23c84b831732e39304c.png with policy now +2023-07-12 08:12:08,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:08,634 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:08,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:08,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:09,048 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5gxbrbrh-media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 08:12:09,083 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3tj95eek-media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 08:12:09,103 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lfxk4m8z-media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 08:12:09,156 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mc42f72a-media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 08:12:09,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:12:09,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:14,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:12:14,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:12:14,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:14,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:15,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:15,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:15,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 08:12:15,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 08:12:15,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 08:12:15,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 08:12:15,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:12:15,875 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xf61teje-media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 08:12:15,905 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grmgs1hj-media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 08:12:15,988 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kizhbh83-media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 08:12:15,988 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yve46ed0-media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 08:12:18,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:20,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:23,335 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:25,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:27,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:27,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:27,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:27,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:29,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:12:29,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:12:29,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:31,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:31,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:31,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:36,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:38,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:38,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:38,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:38,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:39,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:39,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:39,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:12:42,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:45,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:12:45,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:12:45,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:47,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:48,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:48,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:48,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:48,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:48,588 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:53,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:55,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:55,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:58,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:59,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:59,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:59,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:59,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:01,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:01,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:01,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:03,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:03,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:04,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:09,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:09,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:13:10,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:10,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:10,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:10,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:14,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:17,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:17,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:17,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:19,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:19,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:20,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:20,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:20,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:20,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:25,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:27,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:27,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:29,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:29,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:29,920 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:29,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:30,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:32,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:32,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:35,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:35,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:36,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:39,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:13:40,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:40,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:40,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:40,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:42,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:43,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:43,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:44,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:44,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:44,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:44,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:47,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:47,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:47,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:47,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:51,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:51,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:52,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:57,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:02,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:02,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:03,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:08,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:09,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:14:13,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:17,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:17,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:14:17,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:19,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:24,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:29,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:32,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:14:32,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:32,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:34,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:39,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:39,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:14:39,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_e3afb0220a3fb4fd12f3.png with policy now +2023-07-12 08:14:39,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_35975d12907982332b52.png with policy now +2023-07-12 08:14:39,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_e84cb7c3e576b3bd6e96.png with policy now +2023-07-12 08:14:39,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,944 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_e96fba72991bb49f94d6.png with policy now +2023-07-12 08:14:39,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:14:39,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:14:39,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:14:39,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:14:40,227 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iyv8179n-media/images/validation_1960_35975d12907982332b52.png +2023-07-12 08:14:40,243 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lsq0rm8g-media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 08:14:40,260 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rfq4d9jh-media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 08:14:40,294 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c29pgtdo-media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 08:14:44,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:47,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:14:47,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:47,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:47,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:14:47,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:14:47,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 08:14:47,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 08:14:47,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_35975d12907982332b52.png +2023-07-12 08:14:47,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 08:14:47,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:14:48,284 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2yslq4i3-media/images/validation_1960_35975d12907982332b52.png +2023-07-12 08:14:48,329 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/655fjxj5-media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 08:14:49,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:14:49,291 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2hqrfp1j-media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 08:14:49,524 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sl4nhntb-media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 08:14:50,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:55,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:55,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:14:59,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:14:59,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:14:59,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:14:59,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:01,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:02,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:02,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:02,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:03,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:03,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:07,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:09,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:09,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:09,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:09,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:09,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:15:11,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:11,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:12,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:17,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:17,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:17,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:18,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:20,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:20,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:20,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:20,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:20,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:23,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:28,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:28,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:29,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:31,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:31,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:31,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:31,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:32,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:32,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:32,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:34,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:36,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:36,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:39,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:39,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:15:41,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:41,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:41,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:41,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:44,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:44,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:44,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:47,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:47,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:47,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:50,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:50,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:50,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:50,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:50,928 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:52,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:52,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:55,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:00,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:00,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:16:00,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:00,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:16:00,777 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:16:00,779 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:16:03,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:03,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:03,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:05,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:08,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:16:08,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:09,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:16:11,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:16:11,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:11,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:16:11,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:16:11,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:16:16,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:16:16,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:16:16,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:16:16,240 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:16:16,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:16:16,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:16,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:18,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:18,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:18,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:22,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:27,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:32,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:33,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:33,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:38,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:39,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:16:43,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:48,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:48,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:48,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:49,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:54,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:59,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:03,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:17:03,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:03,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:04,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:09,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:09,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:17:10,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_1c5cd897c6da1cf72c60.png with policy now +2023-07-12 08:17:10,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_b693a85e0ba7b021e8b5.png with policy now +2023-07-12 08:17:10,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,949 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_2418660f78d4aba7393a.png with policy now +2023-07-12 08:17:10,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_f777a51b498e8fce06d9.png with policy now +2023-07-12 08:17:11,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:11,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:11,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:11,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:11,252 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/38ay01bz-media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 08:17:11,274 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5n91va6k-media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 08:17:11,301 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dxe2v6sd-media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 08:17:11,307 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3lah2fq-media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 08:17:12,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:12,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 08:17:12,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 08:17:12,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 08:17:12,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 08:17:12,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:17:12,989 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wwehsu26-media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 08:17:12,989 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/idf84ram-media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 08:17:12,998 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ymylitfh-media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 08:17:13,005 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9h5po8xk-media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 08:17:15,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:18,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:17:18,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:18,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:20,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:20,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:20,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:26,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:28,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:29,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:29,832 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:29,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:29,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:31,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:33,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:33,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:17:33,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:36,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:36,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:36,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:39,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:39,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:39,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:39,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:39,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:17:42,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:44,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:44,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:47,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:48,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:48,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:50,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:50,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:50,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:50,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:52,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:52,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:52,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:58,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:59,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:59,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:59,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:59,777 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:00,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:00,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:04,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:04,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:04,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:04,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:08,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:09,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:09,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:09,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:09,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:09,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:09,625 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:18:14,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:16,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:16,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:19,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:19,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:19,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:19,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:19,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:19,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:19,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:19,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:24,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:24,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:25,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:30,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:31,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:31,311 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:31,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:31,313 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:32,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:32,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:34,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:34,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:34,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:36,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:39,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:18:40,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:41,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:41,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:41,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:41,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:41,725 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:46,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:46,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:46,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:46,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:46,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:48,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:48,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:49,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:49,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:49,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:52,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:57,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:02,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:04,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:04,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:04,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:07,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:09,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:19:12,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:17,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:19,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:19,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:19,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:23,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:28,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:33,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:34,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:34,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:34,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:38,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:39,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:19:41,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_1db1fd51970584b5daa1.png with policy now +2023-07-12 08:19:41,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,429 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_7a8ef7854cf8e37f9cf7.png with policy now +2023-07-12 08:19:41,438 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_8caa5ac810c363d03b6e.png with policy now +2023-07-12 08:19:41,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_647b73db1b8d573d85d7.png with policy now +2023-07-12 08:19:41,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:19:41,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:19:41,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:19:41,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:19:41,738 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/66acku56-media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 08:19:41,739 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ny344vyy-media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 08:19:41,775 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xfef5vle-media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 08:19:41,842 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w2c4qdv9-media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 08:19:44,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:45,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:19:45,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:19:45,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 08:19:45,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 08:19:45,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 08:19:45,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 08:19:45,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:19:45,588 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q92b4hie-media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 08:19:45,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8w3ab78-media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 08:19:45,626 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/huj5byyo-media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 08:19:45,631 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fsdjr60a-media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 08:19:49,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:49,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:49,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:49,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:51,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:19:53,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:19:54,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:00,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:00,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:00,700 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:00,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:00,702 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:01,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:01,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:05,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:05,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:05,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:05,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:09,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:09,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:20:10,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:10,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:10,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:10,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:10,777 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:15,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:17,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:17,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:20,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:20,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:20,413 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:20,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:20,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:20,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:21,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:21,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:25,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:25,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:26,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:30,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:30,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:30,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:30,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:31,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:33,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:33,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:35,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:35,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:35,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:37,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:39,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:20:41,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:41,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:41,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:41,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:41,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:42,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:48,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:49,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:49,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:50,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:50,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:50,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:51,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:51,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:51,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:51,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:53,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:57,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:57,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:59,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:02,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:21:02,232 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:21:02,232 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:21:02,234 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:21:04,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:05,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:21:05,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:21:05,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:05,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:05,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:09,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:21:09,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:12,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:21:12,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:21:12,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:21:12,521 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:21:13,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:21:13,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:21:15,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:17,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:21:17,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:21:17,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:21:17,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:21:21,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:21,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:21,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:21,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:21,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:21:21,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:21:26,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:31,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:36,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:36,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:36,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:37,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:39,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:21:42,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:47,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:51,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:51,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:51,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:53,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:58,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:03,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:06,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:22:06,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:06,309 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:08,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:09,634 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:22:11,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_e53b48fcc568a502f437.png with policy now +2023-07-12 08:22:11,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_7e8731b3e3747b303dfe.png with policy now +2023-07-12 08:22:11,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,839 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_283a622f33c91b0b5af0.png with policy now +2023-07-12 08:22:11,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_35002cfdf6c8ccfffdcd.png with policy now +2023-07-12 08:22:11,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:11,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:11,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:11,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:12,141 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f8jrn5lx-media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 08:22:12,192 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lee429j9-media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 08:22:12,194 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eslo6fwy-media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 08:22:12,202 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wu9nbcin-media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 08:22:13,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:17,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:17,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:17,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 08:22:17,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 08:22:17,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 08:22:17,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 08:22:17,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:22:18,148 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q1gyizwa-media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 08:22:18,172 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z4i7bavi-media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 08:22:18,197 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/svvj5qnp-media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 08:22:18,204 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xc7ayvfk-media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 08:22:19,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:21,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:22:21,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:21,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:21,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:24,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:25,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:30,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:30,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:30,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:30,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:30,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:34,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:34,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:35,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:36,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:22:36,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:36,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:39,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:22:40,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:40,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:40,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:40,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:40,791 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:43,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:43,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:46,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:50,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:50,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:50,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:50,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:51,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:51,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:52,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:52,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:52,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:57,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:01,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:01,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:01,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:01,114 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:01,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:01,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:03,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:06,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:06,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:08,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:09,637 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:23:10,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:11,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:11,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:11,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:11,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:13,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:18,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:20,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:20,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:21,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:23:21,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:21,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:22,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:22,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:22,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:22,164 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:24,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:29,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:29,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:29,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:32,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:32,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:32,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:32,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:35,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:37,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:23:37,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:37,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:38,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:38,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:39,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:23:41,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:43,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:43,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:43,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:43,189 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:46,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:47,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:47,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:47,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:47,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:47,804 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:47,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:52,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:52,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:23:52,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:52,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:56,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:57,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:02,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:07,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:07,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:07,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:07,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:09,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:24:13,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:18,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:22,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:22,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:22,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:23,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:28,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:33,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:37,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:37,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:37,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:38,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:39,640 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:24:42,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,523 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_1f7c7153c03240078b4d.png with policy now +2023-07-12 08:24:42,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_56c478641cd971b63c78.png with policy now +2023-07-12 08:24:42,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_0669630fb67332c6ded5.png with policy now +2023-07-12 08:24:42,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_81cc3487bd6c41e95ef1.png with policy now +2023-07-12 08:24:42,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:24:42,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:24:42,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:24:42,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:24:42,830 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dugt5132-media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 08:24:42,866 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lqgmk4cc-media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 08:24:42,873 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/732jlvmw-media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 08:24:42,909 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c822fzfl-media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 08:24:44,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:50,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:50,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:24:50,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:24:50,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 08:24:50,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 08:24:50,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 08:24:50,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 08:24:50,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:24:50,769 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8bojs5u-media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 08:24:50,791 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/amh3kfln-media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 08:24:50,791 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5rd3t4r4-media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 08:24:50,798 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9n8gvit-media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 08:24:52,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:24:52,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:52,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:52,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:55,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:59,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:01,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:01,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:01,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:01,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:01,433 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:07,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:08,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:08,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:08,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:08,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:08,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:09,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:25:11,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:11,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:11,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:11,209 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:13,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:17,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:17,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:18,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:21,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:21,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:21,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:21,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:23,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:23,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:23,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:23,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:26,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:26,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:28,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:31,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:31,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:31,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:31,996 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:33,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:35,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:35,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:38,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:38,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:38,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:39,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:25:39,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:42,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:42,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:42,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:42,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:44,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:44,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:44,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:49,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:52,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:52,558 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:52,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:52,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:53,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:53,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:53,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:53,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:53,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:55,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:00,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:01,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:26:01,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:26:01,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:26:01,882 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:26:02,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:26:02,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:26:06,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:08,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:08,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:08,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:10,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:26:11,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:26:12,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:26:12,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:12,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:26:12,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:26:12,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:26:17,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:26:17,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:26:17,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:26:17,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:26:17,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:20,744 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:26:20,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:26:23,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:23,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:23,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:23,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:29,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:34,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:39,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:39,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:39,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:39,080 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:39,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:26:44,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:49,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:54,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:54,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:54,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:55,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:00,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:05,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:09,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:09,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:09,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:09,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:27:10,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:12,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,202 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_ebb2fde791af714a87fa.png with policy now +2023-07-12 08:27:12,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_fe06a780b0eeb017b688.png with policy now +2023-07-12 08:27:12,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_3246141969b6e42324be.png with policy now +2023-07-12 08:27:12,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,238 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_3fcb388422ea2886399f.png with policy now +2023-07-12 08:27:12,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:12,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:12,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:12,286 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:12,571 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1egk7yr3-media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 08:27:12,572 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v1co0t3j-media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 08:27:12,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ics6e0n5-media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 08:27:12,619 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sza7a92h-media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 08:27:15,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:27:15,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 08:27:15,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 08:27:15,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 08:27:15,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 08:27:15,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:27:15,397 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zeypwevo-media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 08:27:15,424 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mslp45cx-media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 08:27:15,451 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jftzn453-media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 08:27:15,459 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uk9ctn8c-media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 08:27:16,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:21,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:22,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:24,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:24,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:24,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:24,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:27,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:32,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:32,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:32,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:32,413 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:32,415 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:27:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:38,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:39,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:27:39,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:39,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:39,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:42,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:42,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:42,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:42,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:42,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:43,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:51,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:27:51,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:53,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:53,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:53,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:53,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:54,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:55,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:55,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:55,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:59,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:00,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:00,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:02,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:02,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:02,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:03,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:09,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:09,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:09,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:28:10,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:10,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:10,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:28:10,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:13,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:13,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:13,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:13,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:16,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:18,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:18,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:22,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:24,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:24,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:24,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:24,083 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:25,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:25,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:25,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:28:27,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:27,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:28,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:33,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:34,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:34,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:34,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:34,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:36,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:36,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:39,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:39,652 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:28:41,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:41,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:41,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:28:44,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:44,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:44,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:44,866 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:44,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:45,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:45,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:49,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:49,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:49,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:49,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:50,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:54,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:54,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:55,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:56,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:56,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:56,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:00,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:05,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:09,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:29:10,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:11,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:11,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:11,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:16,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:21,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:26,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:26,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:26,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:27,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:32,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:37,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:39,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:29:41,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:41,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:41,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:42,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:43,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_13af1ccf733709899b51.png with policy now +2023-07-12 08:29:43,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,886 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_6f7e3605df1d5a3bcd5d.png with policy now +2023-07-12 08:29:43,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,895 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_ae08a04cbc13fea1f1e4.png with policy now +2023-07-12 08:29:43,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_05878893911aa97a5d5b.png with policy now +2023-07-12 08:29:43,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:29:43,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:29:43,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:29:43,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:29:44,188 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qqc5yul4-media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 08:29:44,202 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gq37ij9o-media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 08:29:44,232 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d2lztrf7-media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 08:29:44,274 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wh4b72jw-media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 08:29:48,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:48,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:29:48,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:29:48,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 08:29:48,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 08:29:48,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 08:29:48,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 08:29:48,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:29:49,073 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zmouv3nn-media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 08:29:49,086 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9n0fqzl4-media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 08:29:49,095 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27i27fau-media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 08:29:49,279 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z61ewems-media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 08:29:53,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:29:54,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:56,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:57,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:29:59,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:03,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:03,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:03,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:03,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:05,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:06,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:06,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:09,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:30:10,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:12,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:30:12,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:12,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:13,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:13,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:13,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:13,293 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:15,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:15,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:15,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:21,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:22,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:22,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:22,985 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:22,987 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:24,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:24,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:26,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:27,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:27,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:30:27,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:31,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:33,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:33,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:33,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:33,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:33,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:33,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:36,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:39,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:30:42,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:42,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:42,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:30:42,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:42,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:43,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:43,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:43,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:43,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:47,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:51,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:51,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:52,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:52,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:52,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:52,804 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:52,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:57,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:57,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:58,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:01,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:31:01,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:03,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:03,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:31:03,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:31:03,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:31:03,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:31:08,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:09,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:31:10,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:31:10,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:13,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:31:13,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:13,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:31:13,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:31:13,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:13,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:31:13,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:31:13,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:31:18,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:31:18,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:31:18,475 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:31:18,477 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:31:18,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:19,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:31:19,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:24,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:28,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:28,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:28,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:31:29,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:34,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:39,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:39,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:31:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:31:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:43,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:31:44,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:49,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:54,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:58,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:31:58,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:58,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:00,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:05,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:09,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:32:10,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:12,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_1821049762b6b5d37c08.png with policy now +2023-07-12 08:32:12,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_e34ade0521fc9adbaae3.png with policy now +2023-07-12 08:32:12,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_d0209e06d5c81f80ec0d.png with policy now +2023-07-12 08:32:12,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_835fcd4e7ec0584630ce.png with policy now +2023-07-12 08:32:12,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:12,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:12,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:12,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:12,705 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/efnvb5db-media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 08:32:12,732 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3frk66le-media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 08:32:12,746 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5b8qkbk3-media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 08:32:12,787 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7fwfjl7i-media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 08:32:13,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:13,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 08:32:13,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 08:32:13,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 08:32:13,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 08:32:13,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:32:13,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:13,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:13,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:13,643 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/95tbbep1-media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 08:32:13,651 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t2xhbjc5-media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 08:32:13,659 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/io0906j1-media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 08:32:13,684 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jv2zq1pt-media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 08:32:16,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:21,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:22,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:23,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:26,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:28,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:28,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:28,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:31,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:31,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:32,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:32,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:32,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:32,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:36,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:39,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:32:40,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:40,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:41,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:42,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:42,425 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:42,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:42,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:43,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:46,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:49,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:49,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:51,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:51,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:51,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:51,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:52,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:58,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:58,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:58,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:59,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:59,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:59,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:02,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:02,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:02,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:02,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:03,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:07,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:07,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:09,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:09,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:33:13,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:13,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:13,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:13,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:14,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:14,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:14,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:14,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:16,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:16,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:20,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:23,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:23,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:23,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:23,576 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:25,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:25,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:25,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:29,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:29,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:29,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:30,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:33,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:33,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:33,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:33,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:35,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:39,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:33:41,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:43,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:44,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:44,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:44,029 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:44,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:44,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:44,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:46,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:48,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:48,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:48,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:48,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:51,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:52,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:52,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:56,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:59,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:59,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:59,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:02,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:07,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:09,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:34:12,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:14,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:14,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:14,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:17,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:22,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:27,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:29,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:29,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:29,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:33,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:38,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:39,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:34:43,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,253 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_6a4be91a47a6698809a9.png with policy now +2023-07-12 08:34:43,262 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,262 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_167355b47a08bf8109e9.png with policy now +2023-07-12 08:34:43,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,272 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_7914a4c603a50f2ea240.png with policy now +2023-07-12 08:34:43,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_6b091a76f28e18553ef9.png with policy now +2023-07-12 08:34:43,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:34:43,323 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:34:43,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:34:43,325 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:34:43,613 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c92ef3gt-media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 08:34:43,638 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tayfn3cy-media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 08:34:43,640 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l5d5njmc-media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 08:34:43,641 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pyohl9t1-media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 08:34:44,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:44,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:44,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:44,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:46,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:34:46,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:34:46,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 08:34:46,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 08:34:46,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 08:34:46,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 08:34:46,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:34:47,326 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wjov2f75-media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 08:34:47,360 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mlrjr6hn-media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 08:34:47,376 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oe744zs2-media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 08:34:47,385 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27dyyobl-media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 08:34:49,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:53,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:34:55,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:55,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:34:59,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:59,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:59,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:00,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:03,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:03,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:03,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:03,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:04,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:04,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:05,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:09,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:35:11,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:12,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:12,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:12,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:12,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:13,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:13,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:14,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:14,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:14,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:16,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:22,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:22,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:22,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:22,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:22,135 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:22,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:22,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:27,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:29,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:29,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:29,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:31,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:31,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:31,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:31,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:31,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:31,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:32,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:38,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:39,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:35:41,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:41,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:41,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:41,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:41,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:44,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:44,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:44,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:44,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:49,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:50,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:50,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:51,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:51,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:51,927 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:51,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:55,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:59,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:59,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:59,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:59,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:59,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:00,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:01,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:36:01,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:36:01,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:36:01,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:36:06,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:08,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:36:08,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:36:09,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:36:11,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:12,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:36:12,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:36:12,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:36:12,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:36:14,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:36:14,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:36:14,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:17,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:36:17,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:36:17,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:17,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:36:17,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:36:17,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:36:17,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:36:23,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:26,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:36:26,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:36:28,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:29,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:36:29,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:36:29,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:33,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:38,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:39,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:36:44,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:44,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:36:44,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:36:44,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:49,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:54,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:59,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:00,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:00,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:05,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:09,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:37:10,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:12,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,076 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_95ba8564905e15131c15.png with policy now +2023-07-12 08:37:12,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_025a92e01716e1b46d6d.png with policy now +2023-07-12 08:37:12,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_d4fb051468326f3eb57a.png with policy now +2023-07-12 08:37:12,106 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,106 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_8ea42d44ae0516e0ca5e.png with policy now +2023-07-12 08:37:12,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:12,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:12,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:12,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:12,403 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sgy05aqm-media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 08:37:12,410 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pb49mgdt-media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 08:37:12,412 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dc09qtzb-media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 08:37:12,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l5eksgej-media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 08:37:15,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:15,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:15,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:16,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:20,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:20,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:20,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 08:37:20,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 08:37:20,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 08:37:20,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 08:37:20,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:37:20,859 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i0egwudq-media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 08:37:20,919 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vrxnypbz-media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 08:37:20,919 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1z6bbz9-media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 08:37:20,959 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/viyihogi-media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 08:37:21,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:22,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:27,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:29,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:30,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:30,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:30,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:31,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:31,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:31,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:31,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:32,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:38,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:38,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:38,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:39,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:37:42,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:42,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:42,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:42,298 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:44,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:45,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:45,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:45,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:47,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:47,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:49,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:52,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:52,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:52,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:52,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:54,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:56,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:56,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:00,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:00,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:02,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:02,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:02,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:02,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:05,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:05,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:05,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:09,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:38:10,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:12,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:12,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:12,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:12,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:14,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:14,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:15,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:15,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:15,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:16,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:22,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:23,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:23,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:23,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:23,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:23,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:23,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:27,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:30,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:30,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:30,324 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:32,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:32,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:33,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:33,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:33,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:33,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:38,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:39,675 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:38:41,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:41,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:43,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:44,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:44,158 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:44,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:44,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:46,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:46,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:46,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:48,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:48,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:48,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:48,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:48,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:50,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:50,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:54,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:59,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:01,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:01,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:01,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:05,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:09,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:39:10,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:15,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:16,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:16,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:16,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:21,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:26,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:31,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:31,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:31,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:32,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:37,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:39,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:39:42,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:43,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_e0fa444f603a66029fbe.png with policy now +2023-07-12 08:39:43,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_d6f502d4c0d1394b0881.png with policy now +2023-07-12 08:39:43,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_ad32613282f23549a8bd.png with policy now +2023-07-12 08:39:43,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_5f6fadce63310a783572.png with policy now +2023-07-12 08:39:43,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:39:43,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:39:43,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:39:43,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:39:43,876 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hwgbq58c-media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 08:39:43,907 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ve5sww1b-media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 08:39:43,912 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/le6uc9w5-media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 08:39:44,056 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qw0drbd0-media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 08:39:45,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:39:45,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 08:39:45,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 08:39:45,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 08:39:45,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 08:39:45,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:39:45,506 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/85ycteza-media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 08:39:45,527 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/afd46fix-media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 08:39:45,528 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4q6q0ve1-media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 08:39:45,552 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/opyo5xo7-media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 08:39:46,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:46,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:46,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:48,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:54,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:39:54,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:54,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:00,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:01,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:01,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:01,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:03,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:03,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:03,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:03,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:03,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:05,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:09,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:40:11,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:12,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:12,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:13,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:13,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:13,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:13,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:17,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:18,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:18,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:18,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:21,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:21,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:22,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:23,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:23,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:23,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:23,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:28,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:30,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:30,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:33,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:33,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:33,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:33,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:34,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:34,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:34,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:34,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:38,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:39,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:39,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:39,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:40:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:44,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:44,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:44,026 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:44,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:48,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:48,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:48,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:48,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:48,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:49,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:55,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:55,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:55,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:55,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:55,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:57,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:57,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:00,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:03,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:03,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:03,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:05,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:41:05,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:41:05,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:41:05,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:41:06,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:06,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:41:06,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:09,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:41:11,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:15,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:15,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:41:15,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:41:15,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:41:15,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:41:17,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:18,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:18,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:18,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:20,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:41:20,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:41:20,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:41:20,505 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:41:23,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:24,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:41:24,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:28,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:33,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:33,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:33,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:33,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:38,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:39,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:41:43,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:48,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:48,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:48,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:49,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:54,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:59,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:03,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:03,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:03,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:05,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:09,682 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:42:10,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:14,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:14,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_19ed657f9c10dbd875ca.png with policy now +2023-07-12 08:42:15,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:15,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_e5f36a2a698bd96464b3.png with policy now +2023-07-12 08:42:15,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:15,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_cf0e421ac16cf53c2d5a.png with policy now +2023-07-12 08:42:15,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:15,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_df0b7fe6b5a5f888eeac.png with policy now +2023-07-12 08:42:15,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:15,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:15,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:15,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:15,357 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gujx60cg-media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 08:42:15,410 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b2sm0i5m-media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 08:42:15,456 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nnjs9jiv-media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 08:42:16,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:16,451 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r5orokgt-media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 08:42:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:18,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:18,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:42:18,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 08:42:18,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 08:42:18,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 08:42:18,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 08:42:18,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:42:19,150 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wtpuf8aw-media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 08:42:19,154 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5st5wuxv-media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 08:42:19,157 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vu717kbe-media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 08:42:19,177 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pjwbs952-media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 08:42:21,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:24,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:26,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:27,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:31,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:34,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:34,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:34,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:34,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:34,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:34,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:34,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:36,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:42:36,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:37,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:39,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:42:42,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:45,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:45,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:45,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:45,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:45,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:42:45,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:49,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:49,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:49,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:53,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:54,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:55,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:55,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:55,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:55,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:58,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:03,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:03,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:04,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:05,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:05,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:05,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:05,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:05,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:05,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:05,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:09,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:09,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:43:12,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:12,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:14,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:16,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:16,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:16,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:16,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:19,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:20,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:20,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:20,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:22,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:22,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:25,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:25,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:25,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:25,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:25,980 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:30,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:31,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:31,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:35,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:35,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:35,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:35,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:35,777 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:35,778 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:35,779 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:36,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:39,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:43:40,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:40,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:41,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:46,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:46,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:46,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:46,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:47,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:49,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:49,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:50,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:50,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:50,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:51,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:51,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:51,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:51,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:52,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:57,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:58,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:58,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:44:02,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:05,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:05,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:05,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:08,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:09,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:44:13,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:18,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:20,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:20,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:20,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:23,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:28,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:33,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:35,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:35,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:35,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:38,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:39,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:44:44,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:45,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_df994cef7f6357e3664e.png with policy now +2023-07-12 08:44:45,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_4c3033cd3536b3f44094.png with policy now +2023-07-12 08:44:45,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_0d50a284079de0683ad8.png with policy now +2023-07-12 08:44:45,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,545 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_7ed8ebb926937feea5e8.png with policy now +2023-07-12 08:44:45,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:44:45,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:44:45,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:44:45,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:44:45,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvetz3gn-media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 08:44:45,844 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tkrytovb-media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 08:44:45,856 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0uo5se8h-media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 08:44:45,863 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ta9l84jk-media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 08:44:50,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:50,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:50,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:50,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:52,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:44:52,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:44:52,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 08:44:52,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 08:44:52,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 08:44:52,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 08:44:52,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:44:52,740 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1a9qrpf0-media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 08:44:52,742 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q8q6tlbw-media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 08:44:52,768 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y7jpnj0w-media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 08:44:52,782 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r9bzqi7h-media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 08:44:56,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:56,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:01,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:01,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:05,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:05,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:05,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:06,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:06,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:06,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:06,117 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:07,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:09,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:45:10,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:10,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:12,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:16,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:16,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:16,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:16,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:18,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:19,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:19,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:20,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:20,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:20,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:23,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:26,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:26,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:26,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:26,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:28,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:28,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:28,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:33,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:35,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:35,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:35,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:36,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:36,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:36,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:36,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:37,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:37,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:39,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:39,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:45:44,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:45,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:45,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:45,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:45,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:46,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:46,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:49,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:50,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:50,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:50,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:55,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:55,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:56,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:56,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:56,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:56,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:01,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:04,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:46:04,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:05,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:05,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:05,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:06,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:06,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:46:06,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:46:06,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:46:06,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:09,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:46:12,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:13,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:46:13,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:17,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:46:17,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:46:17,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:46:17,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:18,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:21,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:21,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:21,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:21,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:46:21,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:46:21,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:46:21,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:46:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:23,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:28,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:31,744 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:33,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:36,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:36,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:36,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:39,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:39,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:46:44,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:49,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:51,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:51,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:51,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:55,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:00,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:05,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:06,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:06,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:09,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:47:10,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:15,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:15,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_acf992ec2207b3ea2d60.png with policy now +2023-07-12 08:47:15,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_f680c2bff58b85913aa3.png with policy now +2023-07-12 08:47:15,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_b72a6b1b7e4871bb5d8f.png with policy now +2023-07-12 08:47:15,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,887 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_c267c69f2a522dc95745.png with policy now +2023-07-12 08:47:15,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:15,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:15,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:15,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:16,172 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0bane34p-media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 08:47:16,236 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqq9ul3r-media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 08:47:16,240 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x10ocwxk-media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 08:47:16,292 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9zsytd4-media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 08:47:16,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:47:16,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 08:47:16,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 08:47:16,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 08:47:16,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 08:47:17,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:47:17,356 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vlmyzwec-media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 08:47:17,370 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eakgf9ix-media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 08:47:17,375 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t765t8wp-media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 08:47:17,385 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mnpxswpq-media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 08:47:20,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:21,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:21,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:47:21,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:26,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:26,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:26,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:31,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:35,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:35,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:35,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:35,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:35,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:36,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:36,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:37,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:39,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:47:42,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:44,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:47:44,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:45,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:45,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:45,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:45,384 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:47,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:51,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:51,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:53,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:47:53,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:53,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:55,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:55,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:55,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:55,905 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:59,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:02,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:02,076 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:05,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:05,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:05,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:05,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:05,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:06,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:06,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:09,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:48:10,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:11,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:11,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:16,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:17,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:17,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:17,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:17,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:20,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:20,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:21,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:48:21,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:21,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:21,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:27,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:27,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:27,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:27,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:27,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:29,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:29,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:33,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:37,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:48:37,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:37,064 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:37,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:37,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:37,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:37,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:38,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:38,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:38,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:39,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:48:44,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:47,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:47,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:47,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:47,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:47,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:49,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:51,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:51,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:51,777 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:51,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:52,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:52,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:48:52,064 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:55,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:56,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:56,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:49:00,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:05,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:07,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:07,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:07,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:09,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:49:10,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:15,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:20,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:22,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:22,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:22,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:26,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:31,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:36,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:37,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:37,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:37,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:39,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:49:41,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:46,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,285 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_ed6b49b703996f58696a.png with policy now +2023-07-12 08:49:46,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_333e872b5de120d26892.png with policy now +2023-07-12 08:49:46,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_38040cf3cd9cbe1bc314.png with policy now +2023-07-12 08:49:46,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,316 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_2c125071cfc5a2fe04e6.png with policy now +2023-07-12 08:49:46,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:49:46,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:49:46,357 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:49:46,359 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:49:46,665 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nk3sofxw-media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 08:49:46,697 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/osat85ab-media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 08:49:46,715 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/488dm2sp-media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 08:49:46,717 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3zq85ocv-media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 08:49:47,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:50,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:49:50,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 08:49:50,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 08:49:50,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 08:49:50,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 08:49:50,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:49:51,022 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jo1s7lhu-media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 08:49:51,034 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hckcq32e-media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 08:49:51,046 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ck3jelf4-media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 08:49:51,047 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ltl0wgua-media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 08:49:52,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:52,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:52,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:53,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:56,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:49:58,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:59,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:03,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:06,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:06,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:06,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:06,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:08,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:50:08,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:08,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:08,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:08,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:09,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:50:09,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:15,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:16,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:16,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:16,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:16,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:17,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:17,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:21,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:23,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:23,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:50:23,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:26,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:27,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:27,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:27,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:27,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:27,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:32,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:35,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:35,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:37,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:37,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:37,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:37,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:37,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:38,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:38,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:39,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:50:43,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:44,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:44,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:47,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:47,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:47,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:47,559 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:48,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:53,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:53,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:53,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:53,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:54,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:58,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:58,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:58,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:58,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:00,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:02,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:02,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:05,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:08,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:08,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:08,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:51:08,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:51:08,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:51:08,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:09,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:51:10,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:11,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:11,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:15,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:19,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:51:19,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:51:19,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:51:19,029 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:20,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:20,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:21,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:23,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:23,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:23,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:51:23,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:51:23,742 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:51:23,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:26,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:29,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:29,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:31,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:36,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:38,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:38,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:51:38,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:39,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:51:41,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:46,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:51,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:53,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:53,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:51:53,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:57,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:02,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:07,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:08,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:08,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:08,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:09,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:52:12,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:17,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:18,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_62fae4a01ba8422d6ed9.png with policy now +2023-07-12 08:52:18,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_008df6c26e19a483defd.png with policy now +2023-07-12 08:52:18,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_3ef95d0764b88f6fbafa.png with policy now +2023-07-12 08:52:18,047 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,047 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_6a7c69b67225aaafd76d.png with policy now +2023-07-12 08:52:18,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:18,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:18,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:18,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:52:18,344 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a4x1847f-media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 08:52:18,348 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5lifjk4z-media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 08:52:18,367 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d6xrhlum-media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 08:52:18,369 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z75j56gn-media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 08:52:23,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:23,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:23,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:23,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 08:52:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 08:52:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 08:52:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:52:24,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p2darzwz-media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 08:52:24,614 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpophh2n-media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 08:52:24,648 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0a75aako-media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 08:52:24,653 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nm0sgz57-media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 08:52:27,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:28,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:33,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:34,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:37,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:37,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:37,335 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:37,336 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:52:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:39,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:39,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:39,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:39,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:52:42,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:52:42,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:44,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:47,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:47,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:47,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:47,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:52:50,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:51,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:52:51,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:54,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:54,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:54,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:56,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:57,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:57,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:57,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:57,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:00,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:00,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:01,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:07,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:07,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:07,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:07,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:07,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:09,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:09,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:09,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:09,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:09,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:09,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:53:12,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:17,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:17,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:17,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:17,698 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:17,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:18,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:18,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:22,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:25,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:25,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:25,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:27,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:27,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:28,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:28,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:28,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:28,262 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:32,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:36,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:36,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:38,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:38,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:38,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:38,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:38,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:39,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:53:40,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:40,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:40,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:43,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:45,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:45,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:48,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:48,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:48,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:48,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:48,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:52,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:52,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:52,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:52,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:53,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:54,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:54,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:55,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:55,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:55,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:59,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:04,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:09,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:09,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:54:10,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:10,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:10,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:14,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:19,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:24,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:25,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:25,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:25,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:29,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:34,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:39,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:39,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:54:40,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:40,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:40,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:44,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:47,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_b4413d53814dba669bcf.png with policy now +2023-07-12 08:54:47,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,165 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_7999661823c5d1a194ca.png with policy now +2023-07-12 08:54:47,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_1d395d84a1aa765d6c1b.png with policy now +2023-07-12 08:54:47,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_fe5c8ad23d0df9553dbf.png with policy now +2023-07-12 08:54:47,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:54:47,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:54:47,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:54:47,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:54:47,501 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/09x6xszy-media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 08:54:47,523 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2dz06vbk-media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 08:54:47,541 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iop8q1kb-media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 08:54:47,657 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ji45pzwh-media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 08:54:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:54:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 08:54:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 08:54:48,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 08:54:48,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 08:54:48,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:54:49,230 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gzfwq09l-media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 08:54:49,235 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1se9atxu-media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 08:54:49,253 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ne18v0qm-media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 08:54:49,253 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ry5wnhqo-media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 08:54:49,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:55,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:55,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:55,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:55,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:56,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:54:57,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:00,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:05,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:05,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:05,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:05,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:06,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:06,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:06,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:09,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:55:10,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:10,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:10,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:11,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:15,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:15,319 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:15,320 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:15,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:15,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:15,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:17,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:23,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:24,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:26,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:26,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:26,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:26,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:26,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:26,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:26,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:28,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:33,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:33,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:33,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:36,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:36,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:36,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:36,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:39,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:39,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:55:41,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:41,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:41,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:42,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:42,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:44,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:47,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:47,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:47,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:47,229 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:49,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:52,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:52,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:55,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:56,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:56,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:56,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:57,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:57,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:57,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:57,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:01,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:01,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:01,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:06,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:08,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:56:08,144 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:56:08,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:56:08,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:09,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:56:10,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:10,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:12,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:12,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:12,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:12,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:17,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:17,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:56:17,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:56:17,859 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:56:17,861 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:19,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:19,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:22,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:22,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:56:22,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:56:22,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:56:22,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:27,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:27,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:27,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:28,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:28,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:28,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:33,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:38,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:39,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:56:42,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:42,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:42,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:43,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:48,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:53,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:57,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:57,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:57,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:58,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:03,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:08,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:09,716 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:57:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:57:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:12,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:13,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:16,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,542 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_1b4ddf79dc604699f630.png with policy now +2023-07-12 08:57:16,551 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_259ec0830802f8053fe5.png with policy now +2023-07-12 08:57:16,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_bb1524320261cb0216db.png with policy now +2023-07-12 08:57:16,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_0099dae297478bd3ce67.png with policy now +2023-07-12 08:57:16,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:16,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:16,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:16,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:16,875 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/50gnvfoy-media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 08:57:16,877 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4aszxl8e-media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 08:57:16,903 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nn7vup5m-media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 08:57:16,935 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3s03mdg-media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 08:57:18,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:22,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:22,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:22,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 08:57:22,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 08:57:22,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 08:57:22,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 08:57:22,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:57:22,837 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pax3wzuv-media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 08:57:22,854 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0aa9eveu-media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 08:57:22,878 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uwy7803t-media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 08:57:22,925 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/72e3ul1v-media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 08:57:24,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:26,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:28,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:57:28,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:28,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:29,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:31,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:34,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:35,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:35,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:35,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:35,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:39,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:57:40,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:40,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:40,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:43,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:43,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:57:43,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:45,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:45,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:45,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:45,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:46,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:49,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:49,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:52,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:55,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:55,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:55,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:55,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:57,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:58,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:58,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:58,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:58,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:02,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:06,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:06,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:06,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:06,023 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:07,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:07,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:08,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:09,720 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:58:13,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:13,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:13,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:13,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:58:16,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:16,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:16,836 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:16,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:16,838 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:18,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:24,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:25,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:25,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:27,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:27,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:27,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:27,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:29,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:29,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:29,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:29,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:58:34,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:34,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:34,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:37,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:37,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:37,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:37,810 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:39,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:58:39,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:43,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:43,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:44,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:44,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:44,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:58:45,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:47,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:47,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:47,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:47,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:50,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:52,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:52,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:52,779 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:52,780 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:52,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:52,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:56,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:59,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:59,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:59,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:01,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:01,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:59:06,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:09,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:59:11,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:14,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:14,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:14,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:17,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:22,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:27,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:29,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:29,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:29,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:32,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:37,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:39,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:59:43,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:44,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:44,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:44,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:47,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_825c97191445f88d10ad.png with policy now +2023-07-12 08:59:47,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_ebe038f09b0a2e5c2779.png with policy now +2023-07-12 08:59:47,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_69c0860f204684fe6617.png with policy now +2023-07-12 08:59:47,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 08:59:47,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 08:59:47,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_a9fe0e5593aba42cc7e6.png with policy now +2023-07-12 08:59:47,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:59:47,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:59:47,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:59:47,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:59:47,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:59:47,431 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/znw8p4f3-media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 08:59:47,442 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fit8o2qw-media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 08:59:47,480 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ll2wqavw-media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 08:59:47,493 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ls1ppj1s-media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 08:59:47,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2qc1coky-media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 08:59:47,893 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqvkn1pr-media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 08:59:49,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:54,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:56,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:59:56,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:59:56,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 08:59:56,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 08:59:56,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:59:56,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:59:56,681 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bjeftafh-media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 08:59:56,713 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/prbunew4-media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 08:59:59,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:59,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:59,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:59,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:04,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:05,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:06,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:06,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:06,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:06,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:09,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:00:10,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:14,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:14,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:15,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:00:15,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:00:15,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:15,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:15,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:15,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:15,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:16,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:22,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:23,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:23,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:26,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:26,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:26,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:26,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:28,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:30,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:00:30,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:00:30,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:32,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:32,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:33,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:35,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:35,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:35,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:35,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:39,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:39,725 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:00:41,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:41,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:44,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:45,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:00:45,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:00:45,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:46,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:46,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:46,286 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:46,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:49,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:50,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:50,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:55,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:56,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:56,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:56,226 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:56,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:59,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:59,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:00,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:00,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:00,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:00,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:06,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:01:06,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:06,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:01:06,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:01:06,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:01:08,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:01:08,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:09,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:01:11,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:16,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:16,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:16,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:16,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:01:16,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:01:16,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:16,983 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:01:16,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:01:17,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:01:17,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:22,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:01:22,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:01:22,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:01:22,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:22,138 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:01:26,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:01:26,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:27,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:31,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:38,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:39,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:01:43,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:46,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:46,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:46,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:49,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:54,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:59,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:01,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:01,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:01,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:05,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:09,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:02:10,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:15,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:16,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:16,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:16,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:16,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_46dad1ead66f697f0698.png with policy now +2023-07-12 09:02:16,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_f451e47ddafe0bf1fcc6.png with policy now +2023-07-12 09:02:16,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_f44d5fe284ffc3b74829.png with policy now +2023-07-12 09:02:16,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_4f03c9db92527c876ba9.png with policy now +2023-07-12 09:02:16,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:16,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:16,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:16,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:17,124 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sb5l3umv-media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 09:02:17,126 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/io2ihe5k-media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 09:02:17,136 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2utpcrkj-media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 09:02:17,139 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q9jbw4kn-media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 09:02:20,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:20,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 09:02:20,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 09:02:20,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 09:02:20,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 09:02:20,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:02:21,071 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mm3dyh0f-media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 09:02:21,090 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m2xznws5-media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 09:02:21,128 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f9dq57u7-media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 09:02:21,148 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bejc8ccj-media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 09:02:21,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:26,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:27,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:29,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:31,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:31,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:36,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:36,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:36,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:36,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:37,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:38,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:38,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:39,729 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:02:42,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:46,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:46,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:46,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:46,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:46,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:46,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:46,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:47,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:47,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:47,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:53,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:56,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:56,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:56,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:56,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:56,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:56,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:59,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:01,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:01,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:01,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:04,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:05,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:06,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:06,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:06,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:06,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:09,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:09,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:03:14,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:14,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:14,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:16,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:16,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:16,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:17,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:17,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:17,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:17,359 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:19,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:24,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:24,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:25,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:27,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:27,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:27,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:27,352 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:30,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:31,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:31,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:31,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:33,044 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:36,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:37,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:37,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:37,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:37,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:39,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:03:41,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:42,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:42,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:47,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:47,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:47,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:47,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:47,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:47,836 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:47,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:47,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:51,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:51,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:52,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:53,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:53,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:53,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:53,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:58,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:00,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:04:00,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:04:02,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:02,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:02,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:04,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:09,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:09,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:04:14,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:17,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:17,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:17,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:20,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:25,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:30,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:32,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:32,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:32,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:35,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:39,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:04:40,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:45,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:47,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_f78f21bafff0abaf0089.png with policy now +2023-07-12 09:04:47,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_1ee652c4e539d1e07cce.png with policy now +2023-07-12 09:04:47,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,198 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_b97ae163d55862639f70.png with policy now +2023-07-12 09:04:47,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_ec40a3df133e538d57f3.png with policy now +2023-07-12 09:04:47,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:04:47,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:04:47,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:04:47,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:04:47,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:47,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:47,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:47,497 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9411z14-media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 09:04:47,516 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/01hwe2qk-media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 09:04:47,523 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3rlhwyqe-media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 09:04:47,603 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7u6x1jrq-media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 09:04:50,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:54,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:04:54,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:04:54,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 09:04:54,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 09:04:54,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 09:04:54,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 09:04:54,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:04:54,714 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y1wi1297-media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 09:04:54,775 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bcj4s9uc-media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 09:04:54,790 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7lldtcou-media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 09:04:54,794 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgcvtx6r-media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 09:04:56,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:58,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:01,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:02,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:05:02,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:02,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:03,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:07,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:07,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:07,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:07,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:07,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:09,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:05:12,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:12,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:12,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:17,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:05:17,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:17,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:17,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:17,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:17,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:17,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:17,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:21,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:21,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:23,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:27,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:27,451 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:27,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:27,453 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:29,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:30,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:30,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:32,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:32,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:34,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:37,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:37,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:37,604 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:37,606 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:39,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:39,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:39,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:05:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:46,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:48,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:05:48,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:48,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:48,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:48,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:48,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:48,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:48,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:52,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:57,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:57,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:57,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:58,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:58,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:58,428 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:58,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:02,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:03,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:03,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:03,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:06,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:06:06,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:08,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:08,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:06:08,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:06:08,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:06:08,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:09,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:06:13,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:15,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:06:15,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:18,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:18,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:18,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:18,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:06:18,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:18,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:06:18,739 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:06:18,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:23,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:06:23,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:06:23,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:06:23,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:24,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:24,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:06:24,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:29,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:33,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:33,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:33,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:33,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:34,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:39,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:39,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:06:44,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:48,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:48,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:48,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:50,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:55,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:00,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:03,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:03,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:03,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:05,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:09,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:07:10,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:15,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:17,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_36b4d3a784d14fee51a0.png with policy now +2023-07-12 09:07:17,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_33d957838c074e90f6a2.png with policy now +2023-07-12 09:07:17,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_533582fdbffc2ddc97ac.png with policy now +2023-07-12 09:07:17,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,684 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_def8ab43612eed95556d.png with policy now +2023-07-12 09:07:17,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:17,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:17,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:17,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:17,979 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q6v4fsdl-media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 09:07:17,986 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wajsl1k1-media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 09:07:17,994 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rvke0zc0-media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 09:07:18,008 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6b9s5mru-media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 09:07:18,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:18,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:18,427 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:19,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:07:19,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 09:07:19,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 09:07:19,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 09:07:19,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 09:07:19,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:07:19,389 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1jdorgt-media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 09:07:19,407 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jx8096u7-media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 09:07:19,449 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i6p9aolc-media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 09:07:19,452 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3mv5yx9m-media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 09:07:21,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:26,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:27,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:28,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:32,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:33,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:33,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:33,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:36,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:36,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:36,334 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:36,336 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:37,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:07:37,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:38,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:39,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:07:43,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:46,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:46,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:46,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:46,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:46,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:48,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:48,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:48,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:48,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:54,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:55,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:07:55,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:56,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:56,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:56,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:56,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:59,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:03,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:03,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:03,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:04,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:04,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:05,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:06,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:06,335 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:06,336 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:06,338 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:09,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:08:10,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:13,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:13,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:15,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:16,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:16,348 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:16,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:16,350 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:18,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:18,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:21,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:22,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:22,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:26,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:26,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:26,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:26,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:26,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:31,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:31,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:32,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:33,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:33,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:33,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:36,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:36,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:36,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:36,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:37,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:39,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:08:40,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:40,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:42,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:46,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:46,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:46,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:46,902 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:47,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:49,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:49,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:49,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:49,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:49,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:51,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:51,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:51,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:51,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:53,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:58,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:58,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:58,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:03,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:04,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:04,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:04,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:09,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:09,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:09:14,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:19,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:19,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:19,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:25,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:30,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:34,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:34,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:34,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:35,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:39,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:09:40,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:45,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:45,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_d63b39c78c18b127efec.png with policy now +2023-07-12 09:09:45,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_f1a0da8e451df830c88e.png with policy now +2023-07-12 09:09:45,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_5d7322114061dc65484d.png with policy now +2023-07-12 09:09:45,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_f5a9c542fcce2ab79bf5.png with policy now +2023-07-12 09:09:45,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:09:45,843 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:09:45,844 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:09:45,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:09:46,105 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rw4c7c7y-media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 09:09:46,108 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4rhba8p-media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 09:09:46,128 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6k36utwc-media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 09:09:46,133 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9c01fehe-media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 09:09:49,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:49,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:49,429 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:50,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 09:09:52,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 09:09:52,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 09:09:52,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:09:53,099 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eqv3xulz-media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 09:09:53,107 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3o8z8ld2-media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 09:09:53,145 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ze9hii2-media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 09:09:53,153 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rkbuiiee-media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 09:09:55,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:09:56,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:01,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:01,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:04,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:04,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:04,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:05,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:05,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:05,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:05,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:07,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:09,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:10:10,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:10,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:12,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:14,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:14,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:14,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:14,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:18,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:19,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:19,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:19,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:19,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:19,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:23,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:25,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:25,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:25,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:25,149 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:28,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:28,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:28,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:34,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:35,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:35,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:35,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:35,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:35,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:35,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:35,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:37,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:37,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:39,749 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:10:39,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:45,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:46,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:46,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:46,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:46,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:46,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:46,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:50,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:50,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:50,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:50,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:56,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:56,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:56,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:56,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:56,353 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:56,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:01,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:05,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:11:05,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:05,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:11:05,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:05,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:06,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:11:06,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:11:06,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:11:06,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:07,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:09,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:11:12,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:14,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:11:14,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:17,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:11:17,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:11:17,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:11:17,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:18,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:20,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:11:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:20,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:22,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:11:22,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:11:22,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:11:22,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:23,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:11:23,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:24,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:29,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:32,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:34,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:35,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:35,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:39,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:39,751 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:11:44,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:49,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:50,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:11:50,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:50,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:55,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:00,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:05,511 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:05,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:09,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:12:10,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:15,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:16,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_b89f7b812f07101e7bc5.png with policy now +2023-07-12 09:12:16,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_51f75fba279dc7d2d626.png with policy now +2023-07-12 09:12:16,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_cd64ca37330df964f268.png with policy now +2023-07-12 09:12:16,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_cd935f0213029522bb6c.png with policy now +2023-07-12 09:12:16,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:16,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:16,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:16,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:12:17,234 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lpfwbjv1-media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 09:12:17,260 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2luf954-media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 09:12:17,293 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1duao47h-media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 09:12:17,308 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6egr8yjc-media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 09:12:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:12:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 09:12:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 09:12:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 09:12:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 09:12:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:12:17,777 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ot3oq1ws-media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 09:12:17,781 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nsc28c29-media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 09:12:17,792 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v9jqywm8-media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 09:12:17,832 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h86z7twp-media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 09:12:20,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:20,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:20,511 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:21,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:26,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:26,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:27,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:32,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:35,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:35,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:35,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:35,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:36,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:36,224 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:36,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:36,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:12:38,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:39,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:12:44,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:44,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:12:44,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:45,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:45,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:45,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:45,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:12:49,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:50,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:50,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:50,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:53,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:12:53,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:54,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:56,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:56,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:56,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:56,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:00,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:02,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:02,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:06,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:06,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:06,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:06,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:08,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:08,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:08,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:08,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:09,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:13:11,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:11,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:13,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:16,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:16,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:16,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:16,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:18,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:20,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:20,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:21,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:21,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:21,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:24,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:27,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:27,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:27,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:27,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:29,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:29,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:30,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:35,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:36,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:36,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:36,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:37,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:37,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:37,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:37,335 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:38,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:38,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:39,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:13:40,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:46,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:47,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:48,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:48,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:48,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:48,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:51,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:52,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:52,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:52,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:53,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:53,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:53,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:53,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:56,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:56,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:57,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:02,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:07,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:07,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:07,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:08,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:09,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:14:13,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:18,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:22,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:22,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:22,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:24,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:29,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:34,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:37,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:37,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:37,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:39,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:14:39,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:44,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:47,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_963ea35b0f8a3f28bf7f.png with policy now +2023-07-12 09:14:47,261 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,261 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_5479be16d55799087c1d.png with policy now +2023-07-12 09:14:47,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,272 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_485475519d4a6c767788.png with policy now +2023-07-12 09:14:47,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,301 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_8e5e62fe39fe252d734c.png with policy now +2023-07-12 09:14:47,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:14:47,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:14:47,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:14:47,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:14:47,566 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1l2s4df6-media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 09:14:47,574 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wg3crt76-media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 09:14:47,596 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uj2ipm5r-media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 09:14:47,632 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/plshgnnc-media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 09:14:50,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:51,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:14:51,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:14:51,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 09:14:51,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 09:14:51,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 09:14:51,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 09:14:51,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:14:51,366 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t6pxjueg-media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 09:14:51,378 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/90nk4zxp-media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 09:14:51,409 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ohfo08na-media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 09:14:51,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/51mg9qmg-media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 09:14:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:52,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:55,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:57,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:00,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:01,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:06,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:06,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:06,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:06,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:07,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:08,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:15:08,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:08,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:09,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:09,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:09,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:15:12,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:15,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:15,896 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:15,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:15,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:17,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:18,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:18,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:23,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:15:23,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:23,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:23,168 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:25,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:25,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:25,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:25,773 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:27,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:27,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:28,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:33,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:36,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:36,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:36,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:36,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:36,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:36,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:38,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:38,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:39,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:39,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:15:44,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:45,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:45,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:45,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:45,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:45,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:50,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:53,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:53,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:54,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:54,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:55,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:55,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:55,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:55,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:55,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:01,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:03,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:03,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:06,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:16:06,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:06,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:16:06,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:16:06,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:08,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:08,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:08,402 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:09,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:16:11,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:12,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:12,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:17,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:17,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:16:17,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:16:17,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:16:17,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:21,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:21,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:21,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:16:21,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:16:21,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:16:21,903 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:22,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:23,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:23,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:23,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:27,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:30,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:30,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:32,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:37,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:38,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:38,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:38,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:39,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:16:42,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:47,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:52,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:53,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:53,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:53,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:58,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:03,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:08,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:08,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:08,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:08,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:09,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:17:13,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:16,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,110 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_3d504d65d8985e43c4df.png with policy now +2023-07-12 09:17:16,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_7e6627623494cc0c7f15.png with policy now +2023-07-12 09:17:16,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_14e783bea121b92109bd.png with policy now +2023-07-12 09:17:16,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_b4ca94832829e41b0eaa.png with policy now +2023-07-12 09:17:16,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:16,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:16,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:16,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:16,460 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zdly73nn-media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 09:17:16,464 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m2i320el-media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 09:17:16,512 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jsjue2x9-media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 09:17:16,572 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c3nfg7yw-media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 09:17:19,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:23,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:23,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:23,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:24,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:17:24,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:24,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 09:17:24,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 09:17:24,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 09:17:24,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 09:17:24,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:17:25,070 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/42q8qagz-media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 09:17:25,075 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/juojg8r7-media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 09:17:25,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:25,089 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8jm7ewna-media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 09:17:25,115 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e3qnwl3s-media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 09:17:26,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:30,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:33,731 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:35,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:35,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:35,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:35,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:36,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:39,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:39,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:39,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:39,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:17:41,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:42,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:17:42,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:45,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:45,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:45,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:45,503 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:47,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:51,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:17:51,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:53,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:54,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:54,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:54,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:55,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:55,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:55,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:55,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:58,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:00,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:00,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:04,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:05,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:05,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:05,820 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:05,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:09,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:18:09,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:09,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:09,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:09,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:09,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:10,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:15,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:15,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:15,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:15,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:15,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:18,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:18,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:20,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:25,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:25,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:25,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:25,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:25,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:25,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:25,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:25,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:28,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:28,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:31,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:35,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:35,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:35,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:35,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:37,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:37,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:37,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:39,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:18:40,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:40,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:40,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:42,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:45,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:45,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:45,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:45,526 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:46,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:46,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:48,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:50,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:50,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:50,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:50,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:54,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:55,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:55,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:55,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:55,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:55,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:59,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:04,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:09,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:09,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:19:10,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:10,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:10,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:14,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:19,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:24,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:25,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:25,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:25,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:29,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:34,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:39,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:39,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:19:40,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:40,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:40,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:44,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:45,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_fd2b6676675f447dd0bb.png with policy now +2023-07-12 09:19:45,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_a2b2236edc7a209199e4.png with policy now +2023-07-12 09:19:45,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_be095f7407792a1d8c2b.png with policy now +2023-07-12 09:19:45,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_7995e8c15e47416a5a98.png with policy now +2023-07-12 09:19:45,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:19:45,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:19:45,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:19:45,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:19:45,528 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/91yrh4m9-media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 09:19:45,542 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3022emnu-media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 09:19:45,560 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g9fmuyjy-media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 09:19:45,572 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2t89gahv-media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 09:19:49,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:19:49,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:19:49,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 09:19:49,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 09:19:49,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 09:19:49,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 09:19:49,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:19:49,686 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/whp60vb0-media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 09:19:49,709 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gz71cxt5-media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 09:19:49,732 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0r8214o1-media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 09:19:49,754 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k6n4kazr-media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 09:19:50,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:54,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:19:55,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:55,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:55,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:55,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:58,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:00,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:04,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:04,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:04,256 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:04,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:06,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:07,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:07,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:09,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:20:10,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:10,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:10,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:11,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:14,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:14,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:14,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:14,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:16,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:16,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:17,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:23,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:24,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:24,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:24,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:24,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:25,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:25,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:25,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:25,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:25,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:28,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:33,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:34,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:35,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:35,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:35,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:35,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:38,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:39,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:20:40,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:40,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:40,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:43,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:43,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:44,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:45,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:45,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:45,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:45,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:50,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:52,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:52,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:55,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:55,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:55,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:55,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:55,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:55,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:55,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:55,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:01,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:01,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:01,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:05,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:21:05,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:21:05,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:21:05,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:06,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:09,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:21:10,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:10,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:10,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:10,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:10,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:12,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:17,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:21:17,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:21:17,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:21:17,338 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:18,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:19,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:19,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:21,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:21:21,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:21:21,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:21:21,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:23,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:26,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:26,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:26,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:28,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:28,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:29,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:34,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:39,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:39,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:21:41,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:41,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:41,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:44,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:49,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:54,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:56,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:56,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:56,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:59,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:04,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:09,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:09,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:22:11,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:11,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:11,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:14,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:16,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_fa6aae454ea563a1568c.png with policy now +2023-07-12 09:22:16,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_bba8aa2011cbb2b9e998.png with policy now +2023-07-12 09:22:16,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_cc1f25e32b609b2e7f13.png with policy now +2023-07-12 09:22:16,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,191 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_c694e9556fc95a54794c.png with policy now +2023-07-12 09:22:16,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:16,232 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:16,232 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:16,234 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:16,501 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/901tao6y-media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 09:22:16,534 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ltr8cau-media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 09:22:16,550 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lz0bh5p4-media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 09:22:16,567 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o173z6gt-media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 09:22:19,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:22,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:22,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:22,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 09:22:22,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 09:22:22,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 09:22:22,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 09:22:22,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:22:23,336 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aptgboqj-media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 09:22:23,341 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g4b3swqe-media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 09:22:23,353 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9n1uhzmk-media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 09:22:23,355 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/khzbviwm-media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 09:22:25,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:26,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:26,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:26,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:26,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:30,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:32,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:36,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:36,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:36,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:36,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:36,023 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:39,779 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:22:41,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:41,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:41,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:41,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:41,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:42,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:45,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:45,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:45,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:45,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:47,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:50,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:50,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:52,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:55,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:55,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:55,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:55,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:56,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:56,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:56,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:58,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:59,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:59,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:03,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:05,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:05,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:05,588 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:05,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:09,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:09,780 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:23:11,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:11,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:11,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:15,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:16,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:16,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:16,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:16,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:17,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:17,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:20,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:26,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:26,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:26,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:26,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:26,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:26,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:26,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:31,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:35,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:35,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:36,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:36,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:36,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:36,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:37,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:39,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:23:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:41,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:42,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:44,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:44,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:47,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:47,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:47,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:47,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:48,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:52,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:52,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:52,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:52,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:53,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:53,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:53,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:56,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:59,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:04,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:09,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:09,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:24:11,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:11,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:11,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:14,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:19,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:24,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:26,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:26,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:26,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:30,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:35,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:39,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:24:40,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:41,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:41,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:41,737 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:45,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:46,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_c76d157bbb397161c214.png with policy now +2023-07-12 09:24:46,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_cbb566c39078908ba8b3.png with policy now +2023-07-12 09:24:46,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,753 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_17e81b81f9d5ecefe6ad.png with policy now +2023-07-12 09:24:46,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_77e784d7b8aacf9759c6.png with policy now +2023-07-12 09:24:46,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:24:46,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:24:46,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:24:46,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:24:47,047 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gu0dv404-media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 09:24:47,057 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yg21fw7i-media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 09:24:47,064 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9b0wvifq-media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 09:24:47,110 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1kas1vxv-media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 09:24:47,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:24:47,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 09:24:47,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 09:24:47,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 09:24:47,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 09:24:47,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:24:48,001 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fya6adsq-media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 09:24:48,002 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhdcjpxn-media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 09:24:48,009 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tvvv6ft1-media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 09:24:48,011 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eu7b8vnz-media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 09:24:50,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:56,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:56,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:24:56,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:56,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:56,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:56,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:02,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:05,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:05,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:05,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:05,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:06,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:06,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:07,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:09,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:25:12,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:12,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:12,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:12,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:15,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:15,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:15,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:15,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:16,674 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:18,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:24,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:26,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:26,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:26,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:26,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:26,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:26,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:27,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:27,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:27,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:29,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:34,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:36,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:37,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:37,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:37,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:37,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:39,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:25:39,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:42,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:42,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:42,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:45,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:46,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:46,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:47,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:47,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:47,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:47,920 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:50,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:56,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:56,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:56,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:58,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:58,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:58,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:58,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:58,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:58,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:58,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:01,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:06,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:26:06,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:26:07,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:09,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:26:09,597 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:26:09,597 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:26:09,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:09,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:26:12,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:13,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:13,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:26:13,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:16,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:26:16,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:26:17,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:19,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:26:19,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:26:19,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:26:19,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:22,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:24,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:26:24,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:26:24,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:26:24,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:26,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:26:26,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:26:28,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:28,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:28,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:33,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:38,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:39,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:26:43,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:26:43,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:43,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:43,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:48,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:53,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:58,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:26:58,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:58,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:59,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:04,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:09,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:09,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:27:13,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:27:13,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:13,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:14,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:18,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_ddcd1a1e8b54f46f7b26.png with policy now +2023-07-12 09:27:18,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,732 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_c1f37bde1c536851ff31.png with policy now +2023-07-12 09:27:18,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png with policy now +2023-07-12 09:27:18,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_84e331f8cb88258072e5.png with policy now +2023-07-12 09:27:18,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:18,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:18,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:18,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:27:19,095 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vpghxriw-media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 09:27:19,098 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1rkjmuhw-media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 09:27:19,113 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ueey357p-media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 09:27:19,129 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cq034kyn-media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 09:27:19,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:25,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:27,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:27:27,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:27,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 09:27:27,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 09:27:27,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 09:27:27,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 09:27:27,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:27:27,676 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qr1w4g10-media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 09:27:27,690 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f4975hn7-media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 09:27:27,694 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fm4miood-media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 09:27:27,701 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c0f2lysp-media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 09:27:28,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:28,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:27:28,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:29,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:31,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:37,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:37,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:38,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:38,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:38,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:38,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:27:39,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:27:42,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:43,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:27:43,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:43,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:47,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:47,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:27:47,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:48,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:48,468 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:48,468 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:48,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:27:53,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:57,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:27:57,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:58,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:58,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:58,400 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:58,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:58,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:58,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:58,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:03,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:07,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:07,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:08,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:09,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:09,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:09,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:09,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:09,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:28:13,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:28:13,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:28:13,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:28:13,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:17,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:17,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:18,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:18,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:18,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:18,985 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:18,987 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:24,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:27,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:27,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:28,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:28:28,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:28:28,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:28:29,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:29,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:29,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:29,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:30,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:35,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:37,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:37,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:39,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:28:40,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:40,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:40,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:40,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:41,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:45,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:28:45,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:28:45,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:28:46,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:47,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:47,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:51,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:51,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:51,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:51,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:52,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:55,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:55,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:55,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:55,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:57,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:57,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:57,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:29:00,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:00,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:00,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:03,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:07,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:29:08,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:09,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:29:13,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:15,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:15,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:15,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:19,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:24,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:29,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:30,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:30,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:30,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:34,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:39,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:39,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:29:44,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:45,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:50,505 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,505 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_1d09dab1e10018c0f5a2.png with policy now +2023-07-12 09:29:50,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:50,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,521 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_f5085a2630d7643ccb37.png with policy now +2023-07-12 09:29:50,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,531 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_ea013c7b85e22fd83c96.png with policy now +2023-07-12 09:29:50,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_f2fc182f190a6e4d3266.png with policy now +2023-07-12 09:29:50,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:29:50,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:29:50,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:29:50,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:29:50,923 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cmh0wjhi-media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 09:29:50,929 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/69ue3b1y-media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 09:29:50,936 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ks1bxxz-media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 09:29:50,959 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h4c9vzbe-media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 09:29:56,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:57,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:29:57,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:29:57,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 09:29:57,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 09:29:57,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 09:29:57,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 09:29:57,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:29:58,378 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hegycz9f-media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 09:29:58,383 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nb54b5zf-media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 09:29:58,387 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f4kd75cm-media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 09:29:58,388 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eutq3b0l-media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 09:30:00,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:00,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:00,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:00,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:01,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:07,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:07,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:09,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:09,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:09,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:09,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:09,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:30:12,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:15,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:15,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:15,741 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:17,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:17,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:18,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:18,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:18,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:18,983 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:18,985 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:23,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:27,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:27,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:29,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:29,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:29,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:29,857 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:29,859 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:32,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:32,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:32,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:35,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:37,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:37,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:39,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:30:39,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:39,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:39,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:39,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:40,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:45,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:47,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:47,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:47,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:47,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:47,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:50,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:50,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:50,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:50,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:51,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:56,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:58,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:58,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:00,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:00,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:00,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:00,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:01,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:02,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:02,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:02,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:07,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:08,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:31:08,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:09,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:31:11,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:11,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:11,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:11,567 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:12,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:17,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:17,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:17,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:17,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:18,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:31:18,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:22,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:22,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:22,620 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:22,622 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:23,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:27,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:27,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:27,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:27,153 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:28,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:31:28,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:28,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:33,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:34,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:38,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:39,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:39,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:31:44,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:48,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:48,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:48,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:50,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:55,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:00,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:03,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:03,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:03,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:06,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:09,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:32:11,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:16,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:18,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:18,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:18,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:21,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_57fbdf8438f8224664e4.png with policy now +2023-07-12 09:32:21,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,746 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_66b7fc2a071b6e92e172.png with policy now +2023-07-12 09:32:21,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,756 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_1db6dc00cd88a44aa4f5.png with policy now +2023-07-12 09:32:21,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_506c3fdf835cc8b9e9bc.png with policy now +2023-07-12 09:32:21,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:21,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:32:21,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:32:21,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:21,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:32:22,041 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1b6g8ee9-media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 09:32:22,054 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dmdn8byr-media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 09:32:22,076 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oxhlr45e-media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 09:32:22,081 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n97yr37c-media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 09:32:27,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:32:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:32:28,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 09:32:28,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 09:32:28,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 09:32:28,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 09:32:28,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:32:28,887 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9724g1r-media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 09:32:28,900 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wdi0ytjs-media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 09:32:28,924 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ekg5or7-media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 09:32:28,962 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yia8inx3-media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 09:32:31,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:33,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:33,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:33,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:33,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:38,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:38,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:32:39,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:32:40,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:40,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:32:40,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:32:40,219 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:32:43,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:48,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:48,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:48,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:48,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:32:48,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:32:49,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:49,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:49,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:32:49,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:32:49,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:32:54,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:58,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:32:58,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:00,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:00,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:00,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:00,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:00,611 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:03,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:03,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:03,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:05,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:08,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:08,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:09,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:33:11,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:12,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:12,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:12,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:12,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:16,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:18,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:18,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:19,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:19,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:19,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:21,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:21,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:21,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:21,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:22,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:28,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:28,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:28,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:32,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:32,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:32,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:32,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:33,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:34,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:34,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:34,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:38,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:38,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:39,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:33:43,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:43,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:43,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:43,570 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:45,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:48,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:48,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:50,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:50,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:50,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:50,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:53,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:53,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:53,890 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:53,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:55,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:58,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:58,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:58,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:58,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:58,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:58,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:34:01,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:05,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:05,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:05,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:06,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:08,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:34:09,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:34:12,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:17,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:20,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:20,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:20,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:22,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:27,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:32,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:36,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:38,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:39,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:34:43,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:48,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:51,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:51,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:51,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:53,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,135 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_d6d98a9fc2b412d909b2.png with policy now +2023-07-12 09:34:53,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_9e0da1558268d346b2ff.png with policy now +2023-07-12 09:34:53,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,154 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_a95379a0e1bb4ee5eefd.png with policy now +2023-07-12 09:34:53,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,165 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_50d6f6aebffcec48f478.png with policy now +2023-07-12 09:34:53,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:34:53,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:34:53,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:34:53,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:34:53,460 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1i9uvnua-media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 09:34:53,473 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/esu3txv3-media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 09:34:53,483 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a6n2e4v5-media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 09:34:53,494 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ky2zepcz-media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 09:34:54,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:59,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:34:59,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:34:59,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 09:34:59,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 09:34:59,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 09:34:59,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 09:34:59,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:34:59,538 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3l0w83xe-media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 09:34:59,542 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n2j4noa5-media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 09:34:59,556 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9slmzl9-media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 09:34:59,557 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/700rdeiv-media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 09:34:59,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:03,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:05,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:06,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:06,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:06,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:09,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:09,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:35:11,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:12,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:12,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:12,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:12,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:16,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:19,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:19,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:22,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:22,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:22,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:22,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:22,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:22,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:22,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:22,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:27,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:29,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:29,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:32,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:33,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:33,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:33,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:37,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:37,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:37,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:37,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:39,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:39,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:39,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:35:42,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:43,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:43,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:43,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:43,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:47,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:49,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:49,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:52,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:52,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:52,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:53,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:53,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:53,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:53,200 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:53,202 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:58,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:59,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:59,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:03,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:03,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:03,320 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:03,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:04,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:07,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:07,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:07,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:09,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:09,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:36:09,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:09,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:36:13,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:13,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:13,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:13,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:14,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:19,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:36:19,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:19,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:23,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:23,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:23,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:23,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:23,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:23,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:23,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:24,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:29,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:29,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:29,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:29,006 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:29,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:36:29,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:30,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:35,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:38,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:38,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:38,283 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:39,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:39,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:36:40,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:45,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:50,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:53,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:53,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:53,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:56,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:01,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:06,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:08,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:08,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:09,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:37:11,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:16,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:21,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:23,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:23,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:23,344 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_dc4de63f9960a0a31886.png with policy now +2023-07-12 09:37:23,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_d8ebb18636c7304a48a5.png with policy now +2023-07-12 09:37:23,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_8ab18a5e3a00fe86d6ef.png with policy now +2023-07-12 09:37:23,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_f6c9e9af5b280aff421b.png with policy now +2023-07-12 09:37:23,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:23,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:37:23,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:37:23,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:37:23,686 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/akepndb8-media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 09:37:23,711 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/11cxsso8-media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 09:37:23,751 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y5axkk39-media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 09:37:23,770 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ig6384fb-media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 09:37:27,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:29,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:37:29,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:37:29,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 09:37:29,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 09:37:29,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 09:37:29,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 09:37:29,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:37:30,226 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9h8h2fhl-media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 09:37:30,229 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7yoqm9p5-media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 09:37:30,230 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bxgdcyvj-media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 09:37:30,327 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/teu5jzdq-media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 09:37:33,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:33,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:38,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:38,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:38,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:39,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:37:39,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:37:43,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:43,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:37:43,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:37:43,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:37:44,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:49,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:49,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:37:49,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:37:53,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:53,235 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:37:53,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:37:53,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:37:53,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:53,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:55,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:59,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:37:59,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:00,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:03,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:03,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:03,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:03,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:05,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:08,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:08,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:09,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:38:09,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:09,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:11,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:13,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:13,425 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:13,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:13,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:16,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:19,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:19,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:22,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:23,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:38:23,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:23,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:23,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:23,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:23,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:23,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:27,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:30,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:30,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:33,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:33,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:33,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:33,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:33,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:38,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:38:38,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:38,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:38,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:39,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:38:40,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:40,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:44,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:44,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:44,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:44,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:44,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:49,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:50,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:50,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:53,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:38:53,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:53,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:54,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:54,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:54,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:54,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:55,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:59,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:59,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:59,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:59,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:39:00,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:39:00,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:39:00,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:05,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:08,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:08,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:08,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:09,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:39:10,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:39:10,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:15,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:20,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:24,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:24,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:24,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:26,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:31,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:36,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:39,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:39,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:39,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:39,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:39:41,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:46,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:51,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:53,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_229851204683a32b894d.png with policy now +2023-07-12 09:39:53,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_b9647059d362b9b7c491.png with policy now +2023-07-12 09:39:53,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png with policy now +2023-07-12 09:39:53,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_abce74fb7ce87882e4ef.png with policy now +2023-07-12 09:39:53,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:39:53,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:39:53,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:39:53,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:39:53,446 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vc0r03p0-media/images/validation_2300_229851204683a32b894d.png +2023-07-12 09:39:53,493 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/06ysbj9c-media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 09:39:53,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r5bmxrmr-media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 09:39:53,500 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8wscchzy-media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 09:39:54,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:54,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:54,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:57,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:00,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:00,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:00,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 09:40:00,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 09:40:00,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_229851204683a32b894d.png +2023-07-12 09:40:00,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 09:40:00,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:40:00,827 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c8l7fjnt-media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 09:40:00,847 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qcyjhn4j-media/images/validation_2300_229851204683a32b894d.png +2023-07-12 09:40:00,869 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fv8ec9yb-media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 09:40:00,887 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yuk3p8ze-media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 09:40:02,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:03,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:09,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:09,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:09,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:09,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:09,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:40:10,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:11,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:11,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:11,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:11,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:14,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:20,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:20,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:20,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:20,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:20,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:20,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:20,999 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:24,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:24,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:24,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:25,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:30,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:30,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:31,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:31,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:31,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:31,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:31,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:37,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:39,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:39,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:39,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:39,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:40:40,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:40,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:41,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:41,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:41,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:41,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:42,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:47,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:50,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:50,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:52,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:52,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:52,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:52,741 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:52,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:54,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:54,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:54,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:58,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:00,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:00,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:02,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:02,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:02,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:02,697 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:03,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:08,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:09,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:41:10,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:10,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:10,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:13,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:13,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:13,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:13,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:14,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:19,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:20,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:20,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:24,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:24,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:24,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:24,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:25,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:25,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:25,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:25,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:28,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:28,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:28,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:28,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:30,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:30,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:30,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:35,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:39,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:41:40,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:40,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:40,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:40,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:41,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:46,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:51,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:55,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:55,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:55,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:57,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:02,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:07,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:09,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:42:10,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:10,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:10,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:42:12,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:17,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:22,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:23,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_0b62299b4392d7dab3b5.png with policy now +2023-07-12 09:42:23,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,565 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_8adc40877878e2c5992a.png with policy now +2023-07-12 09:42:23,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,576 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_b8447334c2fa99557dab.png with policy now +2023-07-12 09:42:23,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_0a4470d06e9e5634fc30.png with policy now +2023-07-12 09:42:23,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:23,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:42:23,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:42:23,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:42:23,907 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aclp9wbq-media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 09:42:23,918 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5lt9lsx8-media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 09:42:23,922 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rgy602es-media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 09:42:23,938 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/40zgd1vr-media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 09:42:25,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:25,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:25,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:42:27,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:31,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:42:31,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:42:31,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:42:31,473 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8sl8k8n3-media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 09:42:31,487 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ch6dx0s-media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 09:42:31,489 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/csljihcl-media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 09:42:31,559 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pckir8mc-media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 09:42:33,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:33,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:38,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:39,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:42:40,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:40,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:40,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:42:41,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:42:42,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:42,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:42:42,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:42:42,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:42:44,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:49,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:51,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:42:51,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:42:52,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:52,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:42:52,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:42:52,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:42:54,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:55,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:55,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:55,319 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:00,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:01,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:01,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:03,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:03,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:03,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:03,097 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:05,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:09,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:43:10,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:10,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:10,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:10,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:11,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:11,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:13,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:13,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:13,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:13,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:15,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:20,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:21,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:21,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:23,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:23,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:23,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:23,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:25,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:25,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:25,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:25,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:31,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:31,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:31,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:33,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:33,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:33,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:33,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:36,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:39,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:43:40,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:40,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:40,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:41,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:41,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:42,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:45,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:45,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:45,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:45,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:47,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:51,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:51,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:53,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:55,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:55,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:55,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:55,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:55,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:55,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:55,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:58,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:00,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:44:00,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:44:00,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:44:00,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:44:01,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:44:01,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:44:04,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:09,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:09,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:44:10,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:10,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:10,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:14,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:19,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:24,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:25,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:25,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:25,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:29,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:34,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:39,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:44:40,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:40,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:40,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:40,845 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:45,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:50,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:54,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,863 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_6e0bdeefa42f0abef07e.png with policy now +2023-07-12 09:44:54,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,872 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_cad2712e0579795eae61.png with policy now +2023-07-12 09:44:54,883 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_8982b4ca24e18e3a2e66.png with policy now +2023-07-12 09:44:54,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_d536893289030f727428.png with policy now +2023-07-12 09:44:54,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:44:54,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:44:54,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:44:54,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:44:55,167 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s7xlp9ty-media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 09:44:55,176 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/czq0v793-media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 09:44:55,198 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g4kogq33-media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 09:44:55,257 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/43tild5v-media/images/validation_2320_d536893289030f727428.png +2023-07-12 09:44:55,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:55,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:55,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:56,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:01,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:01,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:01,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 09:45:01,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 09:45:01,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_d536893289030f727428.png +2023-07-12 09:45:01,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 09:45:01,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:45:02,338 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sw9twdfs-media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 09:45:02,347 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hsawhjl4-media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 09:45:02,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:02,358 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/31u2wnm1-media/images/validation_2320_d536893289030f727428.png +2023-07-12 09:45:02,382 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q1w72r7f-media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 09:45:04,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:07,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:09,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:45:10,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:10,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:10,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:11,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:12,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:14,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:14,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:14,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:14,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:18,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:21,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:21,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:23,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:24,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:24,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:24,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:24,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:26,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:26,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:26,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:29,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:32,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:32,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:34,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:34,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:34,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:34,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:34,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:39,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:45:40,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:41,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:41,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:41,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:42,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:42,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:45,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:45,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:45,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:45,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:46,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:51,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:52,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:52,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:56,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:56,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:56,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:56,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:56,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:56,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:56,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:56,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:02,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:02,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:02,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:06,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:06,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:06,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:06,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:07,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:09,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:46:11,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:11,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:11,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:12,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:12,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:12,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:17,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:17,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:17,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:17,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:18,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:22,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:22,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:23,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:27,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:27,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:27,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:27,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:27,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:28,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:32,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:32,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:32,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:32,293 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:32,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:32,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:33,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:38,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:39,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:46:42,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:42,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:42,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:42,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:44,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:49,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:54,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:57,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:57,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:57,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:59,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:04,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:09,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:09,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:47:12,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:47:12,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:12,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:15,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:20,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:25,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:26,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_9d435e637b3005c16f38.png with policy now +2023-07-12 09:47:26,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_c9b9321adf648747a0cb.png with policy now +2023-07-12 09:47:26,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,956 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_86f3d365c9fc4e6ab5e8.png with policy now +2023-07-12 09:47:26,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_1afc2708fa415a3e0f40.png with policy now +2023-07-12 09:47:27,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:27,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:47:27,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:47:27,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:47:27,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:47:27,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:27,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:27,278 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5r2h5c07-media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 09:47:27,282 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mr2yrq4n-media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 09:47:27,335 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/806apnxq-media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 09:47:27,337 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kcrcqqjk-media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 09:47:30,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:32,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:47:32,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:47:32,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 09:47:32,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 09:47:32,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 09:47:32,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 09:47:32,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:47:32,961 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bx28zwbj-media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 09:47:32,963 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3aixjuw0-media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 09:47:32,966 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/msm9ebru-media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 09:47:32,973 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gzythlgy-media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 09:47:36,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:36,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:39,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:47:41,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:42,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:42,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:42,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:47:45,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:45,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:47:45,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:47:45,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:47:46,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:52,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:47:52,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:47:52,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:56,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:56,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:47:56,168 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:47:56,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:47:57,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:47:57,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:57,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:57,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:02,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:02,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:03,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:06,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:06,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:06,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:06,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:08,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:09,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:48:12,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:12,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:13,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:48:13,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:13,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:13,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:17,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:17,348 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:17,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:17,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:19,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:22,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:22,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:25,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:28,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:28,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:28,449 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:28,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:28,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:28,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:30,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:32,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:32,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:36,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:38,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:38,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:38,541 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:38,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:39,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:48:41,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:42,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:42,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:43,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:43,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:46,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:48,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:48,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:48,665 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:48,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:52,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:52,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:52,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:57,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:58,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:58,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:59,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:59,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:59,032 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:59,034 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:49:02,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:49:02,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:49:03,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:03,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:49:03,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:49:03,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:49:03,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:49:09,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:09,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:49:12,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:49:12,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:49:13,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:13,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:13,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:49:14,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:19,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:24,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:28,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:28,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:28,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:49:30,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:35,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:39,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:49:40,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:44,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:44,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:44,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:49:46,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:51,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:56,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:58,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_e1920e85b34fdfb6609a.png with policy now +2023-07-12 09:49:58,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_65d2e2217439693f9468.png with policy now +2023-07-12 09:49:58,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_cfd6e181a1bb997a54d6.png with policy now +2023-07-12 09:49:58,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_d7a930a7a71d94c1211f.png with policy now +2023-07-12 09:49:58,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:49:58,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:49:58,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:49:58,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:49:58,512 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vf7yltlz-media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 09:49:58,533 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7k2p6pbh-media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 09:49:58,545 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9yqsklc1-media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 09:49:58,546 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/72epqs35-media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 09:49:59,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:59,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:59,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:01,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:03,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 09:50:03,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 09:50:03,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:50:03,594 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aura8mrm-media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 09:50:03,608 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8hgcvbik-media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 09:50:03,613 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rjr4z1o3-media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 09:50:03,751 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/njlkvtj9-media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 09:50:06,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:07,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:09,843 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:50:12,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:13,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:14,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:50:14,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:50:14,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:16,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:16,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:16,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:16,600 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:18,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:23,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:23,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:23,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:27,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:27,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:27,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:27,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:29,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:29,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:50:29,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:50:29,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:33,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:33,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:34,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:37,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:37,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:37,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:37,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:39,844 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:50:39,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:43,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:43,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:44,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:50:44,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:50:44,876 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:44,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:48,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:48,008 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:48,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:48,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:50,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:53,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:53,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:55,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:58,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:58,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:58,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:58,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:00,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:00,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:51:00,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:00,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:03,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:03,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:05,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:09,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:09,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:09,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:09,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:09,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:51:10,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:13,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:13,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:16,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:51:16,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:16,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:16,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:19,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:19,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:19,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:19,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:21,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:23,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:23,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:27,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:30,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:30,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:30,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:30,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:31,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:31,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:32,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:33,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:33,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:35,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:35,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:35,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:35,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:37,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:39,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:51:42,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:43,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:43,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:46,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:51:46,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:46,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:48,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:53,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:58,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:01,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:01,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:01,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:04,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:09,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:09,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:52:14,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:16,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:16,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:16,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:20,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:25,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:29,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,570 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_576ed58f083c4925ff4f.png with policy now +2023-07-12 09:52:29,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_6b91f2effcbccf59aafb.png with policy now +2023-07-12 09:52:29,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_7116b69ba382286251a6.png with policy now +2023-07-12 09:52:29,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_4776174cbccf4dcad7cf.png with policy now +2023-07-12 09:52:29,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:29,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:52:29,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:52:29,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:52:29,873 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/skd1n6u1-media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 09:52:29,892 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xmow84zg-media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 09:52:29,902 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dr9q5jht-media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 09:52:29,920 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y4dktnl2-media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 09:52:30,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:32,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:32,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:32,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:33,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:52:33,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 09:52:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 09:52:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 09:52:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 09:52:33,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:52:34,363 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m073465u-media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 09:52:34,382 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pvcuhayg-media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 09:52:34,391 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pysuamh9-media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 09:52:34,392 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jnyvuvr6-media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 09:52:35,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:39,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:39,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:52:41,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:43,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:52:47,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:47,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:47,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:47,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:49,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:49,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:52:49,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:52:49,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:52:52,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:54,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:52:54,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:52:57,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:59,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:59,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:52:59,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:52:59,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:02,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:02,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:02,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:03,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:04,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:04,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:08,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:08,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:08,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:08,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:08,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:09,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:53:13,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:14,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:14,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:17,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:17,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:17,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:19,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:19,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:19,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:19,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:19,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:24,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:24,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:24,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:29,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:30,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:30,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:30,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:30,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:32,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:32,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:32,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:35,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:39,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:53:40,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:40,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:40,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:40,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:40,738 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:44,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:44,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:46,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:48,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:48,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:48,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:51,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:52,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:52,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:52,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:52,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:57,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:02,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:54:02,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:54:02,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:54:02,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:54:03,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:03,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:03,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:03,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:04,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:54:04,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:54:06,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:54:06,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:54:06,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:54:06,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:54:08,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:09,853 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:54:13,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:14,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:54:14,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:54:18,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:19,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:19,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:19,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:24,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:29,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:34,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:34,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:34,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:34,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:39,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:39,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:54:44,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:49,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:49,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:49,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:50,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:55,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:00,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:00,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_3d170b31330953e821ba.png with policy now +2023-07-12 09:55:00,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,860 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_6f1a9a30ce0054e1f213.png with policy now +2023-07-12 09:55:00,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_a603ae3e847019132432.png with policy now +2023-07-12 09:55:00,879 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,880 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_4eb285dbb680a44d3c77.png with policy now +2023-07-12 09:55:00,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:00,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:00,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:00,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:01,163 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dg17jeso-media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 09:55:01,211 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xrd8kp3r-media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 09:55:01,214 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o4z0ey59-media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 09:55:01,215 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/btb36lsl-media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 09:55:04,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:04,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:55:04,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:04,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:04,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:04,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 09:55:04,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 09:55:04,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 09:55:04,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 09:55:04,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:55:04,976 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/useq9ui0-media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 09:55:04,978 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0khbs5bd-media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 09:55:04,989 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a09ls7b4-media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 09:55:04,993 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v1zi4ubv-media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 09:55:05,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:09,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:55:10,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:10,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:14,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:15,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:19,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:19,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:55:19,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:19,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:19,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:19,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:19,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:20,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:24,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:24,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:26,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:29,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:29,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:29,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:29,709 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:32,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:34,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:34,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:34,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:34,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:38,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:39,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:55:40,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:40,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:40,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:40,117 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:43,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:44,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:44,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:49,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:49,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:55:49,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:49,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:50,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:50,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:50,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:50,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:54,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:54,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:54,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:00,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:01,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:01,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:01,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:01,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:04,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:04,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:05,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:05,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:05,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:05,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:09,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:56:10,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:11,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:11,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:11,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:11,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:14,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:14,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:16,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:21,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:21,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:21,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:21,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:21,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:21,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:21,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:21,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:24,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:24,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:27,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:31,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:31,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:31,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:31,849 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:32,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:34,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:34,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:36,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:36,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:36,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:36,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:36,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:36,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:36,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:37,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:39,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:56:43,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:48,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:51,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:51,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:51,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:54,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:59,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:04,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:06,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:06,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:06,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:09,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:09,861 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:57:14,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:19,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:21,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:21,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:21,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:25,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:30,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:30,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_6b60edf29451c64b6e2b.png with policy now +2023-07-12 09:57:30,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_e3cc962dadab1208b51d.png with policy now +2023-07-12 09:57:30,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_4fd19c0f067294c7a9e6.png with policy now +2023-07-12 09:57:30,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_81faade70bf66d35a629.png with policy now +2023-07-12 09:57:30,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:57:30,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:57:30,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:57:30,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:57:31,080 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bhv1snwz-media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 09:57:31,081 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7d3651py-media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 09:57:31,119 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ygpr01p1-media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 09:57:31,256 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rkhebevp-media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 09:57:35,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:57:35,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:57:35,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 09:57:35,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 09:57:35,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 09:57:35,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 09:57:35,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:57:35,661 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1gheoe3b-media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 09:57:35,688 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tl2myosp-media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 09:57:35,692 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyb2dkpf-media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 09:57:35,694 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/88l6es9n-media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 09:57:35,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:36,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:36,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:36,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:39,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:57:40,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:57:41,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:45,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:57:47,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:50,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:57:50,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:57:50,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:57:50,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:57:52,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:52,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:52,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:52,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:55,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:57:55,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:57:57,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:00,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:00,980 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:00,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:00,982 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:02,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:05,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:05,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:07,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:07,789 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:08,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:09,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:58:10,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:10,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:10,505 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:10,507 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:13,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:15,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:15,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:18,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:20,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:20,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:20,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:20,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:22,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:22,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:24,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:25,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:25,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:29,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:30,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:30,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:30,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:30,905 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:34,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:35,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:35,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:38,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:58:38,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:38,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:39,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:58:40,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:41,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:41,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:41,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:41,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:45,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:45,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:45,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:51,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:51,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:51,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:51,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:51,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:53,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:58:53,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:53,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:55,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:55,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:57,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:02,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:59:02,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:59:02,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:59:02,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:59:03,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:05,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:59:05,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:59:06,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:59:06,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:59:06,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:59:06,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:59:08,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:08,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:08,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:08,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:09,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:59:13,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:15,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:59:15,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:59:18,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:23,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:23,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:23,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:24,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:29,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:34,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:38,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:38,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:38,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:39,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:39,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:59:44,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:49,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:53,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:53,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:53,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:55,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:00,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:01,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_841e12bb766b52954d05.png with policy now +2023-07-12 10:00:01,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,419 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_3b3663922cc96b138671.png with policy now +2023-07-12 10:00:01,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,429 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_f14220ee5227d3539481.png with policy now +2023-07-12 10:00:01,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_b3144dac275fe828dece.png with policy now +2023-07-12 10:00:01,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:01,564 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:01,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:01,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:01,717 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zn6xdbwo-media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 10:00:01,739 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y4atd9sl-media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 10:00:01,741 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rk6fjqel-media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 10:00:01,830 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uxrazp87-media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 10:00:05,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:05,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:05,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 10:00:05,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 10:00:05,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 10:00:05,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 10:00:05,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:00:06,314 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9xihk05d-media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 10:00:06,331 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v2ksh9r5-media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 10:00:06,333 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2d1d8rw2-media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 10:00:06,341 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/46dzcq2k-media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 10:00:09,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:09,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:09,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:09,868 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:00:11,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:12,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:15,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:17,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:21,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:21,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:21,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:21,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:22,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:24,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:24,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:24,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:25,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:25,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:28,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:31,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:31,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:31,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:31,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:33,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:36,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:36,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:39,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:39,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:39,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:39,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:39,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:00:41,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:41,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:41,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:41,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:45,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:46,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:46,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:50,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:51,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:51,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:51,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:51,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:54,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:54,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:54,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:56,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:56,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:56,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:01,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:01,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:01,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:01,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:01,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:06,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:06,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:07,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:09,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:09,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:09,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:09,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:01:12,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:12,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:12,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:12,444 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:12,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:16,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:16,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:17,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:22,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:22,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:22,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:22,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:23,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:24,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:24,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:24,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:26,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:26,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:28,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:32,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:32,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:32,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:32,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:33,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:36,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:36,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:37,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:37,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:37,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:37,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:39,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:39,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:39,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:39,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:39,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:01:44,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:46,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:46,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:49,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:54,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:54,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:54,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:55,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:00,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:05,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:09,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:09,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:09,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:09,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:02:10,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:15,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:20,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:24,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:24,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:24,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:26,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:31,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:31,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_34c10cf3cd6d89963a05.png with policy now +2023-07-12 10:02:31,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_92717b0771dd961acad8.png with policy now +2023-07-12 10:02:31,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,884 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_674a3618e81973b0470a.png with policy now +2023-07-12 10:02:31,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_0c825ba6ed95b9246d26.png with policy now +2023-07-12 10:02:31,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:02:31,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:02:31,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:02:31,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:02:32,103 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wttkcjw5-media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 10:02:32,199 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ryuwmiw-media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 10:02:32,209 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27tzb0cd-media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 10:02:32,228 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r3v6r8lm-media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 10:02:36,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:02:36,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:02:36,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 10:02:36,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 10:02:36,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 10:02:36,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 10:02:36,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:02:36,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:36,968 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ptmgkva1-media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 10:02:37,020 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ou9mjgek-media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 10:02:37,021 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wd5990ky-media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 10:02:37,046 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpguz4xa-media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 10:02:39,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:39,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:39,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:39,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:02:41,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:02:42,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:46,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:02:47,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:50,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:02:50,879 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:02:50,879 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:02:50,881 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:02:52,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:54,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:54,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:54,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:56,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:02:56,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:02:58,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:01,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:01,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:01,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:01,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:04,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:06,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:06,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:09,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:09,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:09,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:09,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:03:09,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:11,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:11,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:11,095 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:11,097 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:15,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:16,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:16,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:21,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:21,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:21,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:21,098 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:21,100 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:24,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:24,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:24,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:26,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:26,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:26,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:31,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:31,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:31,300 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:31,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:31,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:36,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:36,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:36,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:39,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:39,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:39,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:39,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:03:41,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:41,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:41,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:41,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:42,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:46,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:46,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:47,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:51,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:51,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:51,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:51,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:53,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:54,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:54,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:54,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:56,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:56,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:58,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:01,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:04:01,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:04:01,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:04:01,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:04:03,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:05,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:04:05,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:04:05,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:04:05,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:04:06,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:04:06,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:04:08,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:09,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:09,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:09,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:09,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:04:13,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:17,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:04:18,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:23,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:24,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:24,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:24,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:29,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:34,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:39,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:39,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:39,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:39,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:04:39,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:44,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:49,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:54,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:54,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:54,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:55,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:00,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:00,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_7fcaf32b847afe6bc2ca.png with policy now +2023-07-12 10:05:00,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_3cef30f8c2574b5c86c0.png with policy now +2023-07-12 10:05:00,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png with policy now +2023-07-12 10:05:00,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,709 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_a254fe937a4ee82dda50.png with policy now +2023-07-12 10:05:00,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:00,749 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:00,749 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:00,751 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:00,927 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i3o54obv-media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 10:05:01,003 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5o8cho5b-media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 10:05:01,014 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q0cxw1c8-media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 10:05:01,015 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/46qvkwex-media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 10:05:06,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:07,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:07,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:07,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 10:05:07,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 10:05:07,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 10:05:07,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 10:05:07,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:05:07,689 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qqeeham1-media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 10:05:07,690 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cmla0q6e-media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 10:05:07,722 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t3edz0ov-media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 10:05:07,752 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t8li6o3u-media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 10:05:09,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:09,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:09,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:05:11,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:12,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:17,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:17,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:20,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:20,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:20,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:20,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:22,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:24,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:05:24,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:24,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:27,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:27,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:28,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:30,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:30,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:30,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:30,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:34,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:37,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:37,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:39,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:05:39,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:39,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:39,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:39,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:05:40,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:40,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:40,032 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:40,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:44,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:47,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:47,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:49,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:49,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:49,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:49,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:50,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:54,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:05:54,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:54,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:55,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:57,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:57,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:59,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:59,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:59,737 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:59,739 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:01,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:07,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:07,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:07,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:09,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:09,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:09,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:09,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:06:10,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:10,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:10,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:10,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:12,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:17,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:17,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:17,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:19,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:19,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:19,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:19,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:22,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:24,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:24,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:24,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:27,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:27,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:28,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:30,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:30,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:30,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:30,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:34,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:35,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:35,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:35,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:35,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:37,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:37,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:39,883 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:06:39,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:39,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:39,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:39,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:45,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:50,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:54,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:54,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:54,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:56,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:01,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:06,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:09,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:07:09,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:09,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:09,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:12,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:17,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:22,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:24,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:25,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:25,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:29,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_f055f10be828e22bd53d.png with policy now +2023-07-12 10:07:29,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,854 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_fcac0499a5c4fff7482f.png with policy now +2023-07-12 10:07:29,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_5eb323fa89395b0ed41c.png with policy now +2023-07-12 10:07:29,875 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_5870baca82c8f1ea9c65.png with policy now +2023-07-12 10:07:29,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:29,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:07:29,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:07:29,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:07:30,183 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dp1ixhi3-media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 10:07:30,188 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fsaekjnv-media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 10:07:30,192 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/egl4orij-media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 10:07:30,298 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ck4mh8lu-media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 10:07:32,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 10:07:37,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 10:07:37,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:07:37,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:38,326 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lspb0yyj-media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 10:07:38,349 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yq5v6xcv-media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 10:07:38,443 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wxkn6c5q-media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 10:07:38,549 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r34l3gke-media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 10:07:39,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:07:40,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:40,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:40,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:40,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:43,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:47,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:07:48,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:49,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:49,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:07:49,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:07:49,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:07:54,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:55,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:55,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:55,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:58,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:07:58,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:07:58,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:58,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:07:58,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:07:58,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:07:59,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:05,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:08,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:08,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:08,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:08,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:08,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:08,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:09,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:08:10,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:10,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:10,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:11,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:17,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:18,044 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:18,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:18,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:18,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:18,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:18,804 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:22,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:26,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:26,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:26,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:28,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:28,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:28,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:29,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:29,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:29,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:29,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:33,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:38,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:38,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:39,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:39,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:08:40,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:40,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:40,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:40,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:41,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:41,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:41,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:45,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:48,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:48,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:50,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:50,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:50,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:50,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:51,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:56,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:56,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:56,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:57,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:58,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:58,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:09:01,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:09:01,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:09:01,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:09:01,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:09:03,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:05,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:09:05,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:09:05,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:09:05,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:09:08,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:09:08,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:09:08,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:09,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:09:11,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:11,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:11,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:09:14,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:19,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:24,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:26,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:26,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:26,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:09:30,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:35,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:39,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:09:40,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:41,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:41,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:41,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:09:46,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:51,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:56,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:56,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:56,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:56,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:00,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_e300a087119470bd19f7.png with policy now +2023-07-12 10:10:00,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_a3ef41aa53f7542e2ff6.png with policy now +2023-07-12 10:10:00,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_86a10e9e981b83c54181.png with policy now +2023-07-12 10:10:00,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,198 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_508d57a7914d9268a798.png with policy now +2023-07-12 10:10:00,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:00,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:00,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:00,240 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:00,523 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qnrnjmao-media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 10:10:00,532 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u01fd0md-media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 10:10:00,538 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jdmdtbpw-media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 10:10:00,545 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bc1an829-media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 10:10:02,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:07,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:08,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:08,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:08,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 10:10:08,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 10:10:08,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 10:10:08,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 10:10:08,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:10:09,047 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xn35i7dg-media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 10:10:09,051 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xxqyv611-media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 10:10:09,061 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x97hkk2m-media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 10:10:09,086 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/covzlcdh-media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 10:10:09,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:10:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:11,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:11,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:11,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:12,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:17,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:18,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:19,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:19,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:19,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:19,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:23,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:27,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:27,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:27,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:28,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:28,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:28,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:29,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:29,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:29,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:29,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:34,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:38,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:38,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:39,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:39,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:10:40,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:40,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:40,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:40,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:42,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:42,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:42,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:45,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:48,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:48,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:49,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:49,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:49,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:49,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:50,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:56,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:57,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:57,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:57,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:58,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:58,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:00,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:00,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:00,820 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:00,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:01,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:07,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:08,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:08,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:09,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:11:10,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:10,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:10,995 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:10,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:12,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:12,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:12,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:11:12,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:18,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:19,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:19,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:21,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:21,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:21,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:21,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:23,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:27,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:27,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:27,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:11:28,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:29,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:29,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:31,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:31,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:31,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:31,352 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:34,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:36,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:36,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:36,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:36,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:39,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:39,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:39,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:11:39,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:42,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:42,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:42,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:11:45,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:50,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:55,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:57,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:57,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:57,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:00,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:05,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:09,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:12:10,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:12,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:12,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:12,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:16,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:21,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:26,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:27,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:27,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:27,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:30,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,149 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_d23160faa6354332ac90.png with policy now +2023-07-12 10:12:30,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_e3cbf88a5d454b013c98.png with policy now +2023-07-12 10:12:30,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,167 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_bdfc2304f95d55e3c519.png with policy now +2023-07-12 10:12:30,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,177 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_9d7fc07bee22787a2007.png with policy now +2023-07-12 10:12:30,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:30,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:12:30,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:12:30,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:12:30,484 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5egvasl1-media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 10:12:30,493 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9w1h2neg-media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 10:12:30,553 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vd6lcxqk-media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 10:12:30,556 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2flwhrxk-media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 10:12:32,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:37,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:39,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:12:39,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:12:39,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 10:12:39,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 10:12:39,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 10:12:39,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 10:12:39,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:12:39,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:39,745 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u5nk2mcf-media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 10:12:39,759 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/60vzgurn-media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 10:12:39,774 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a3r605hr-media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 10:12:39,850 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hvn9klte-media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 10:12:39,898 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:12:42,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:42,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:42,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:42,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:48,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:48,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:48,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:12:48,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:12:48,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:12:49,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:12:49,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:12:53,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:57,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:57,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:57,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:58,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:58,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:12:58,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:12:58,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:12:59,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:12:59,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:12:59,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:05,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:07,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:07,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:07,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:07,850 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:09,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:09,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:09,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:13:10,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:13,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:13,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:13,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:15,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:19,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:19,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:19,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:19,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:19,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:21,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:26,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:29,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:29,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:29,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:29,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:29,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:29,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:29,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:29,587 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:29,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:31,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:36,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:39,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:39,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:39,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:13:40,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:40,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:40,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:40,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:42,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:44,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:44,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:44,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:47,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:49,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:49,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:50,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:50,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:50,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:50,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:52,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:57,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:59,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:59,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:59,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:59,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:59,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:14:00,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:14:00,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:14:00,535 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:14:00,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:14:03,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:05,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:14:05,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:14:05,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:14:05,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:14:08,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:09,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:14:09,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:14:09,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:14:13,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:14,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:14,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:14,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:19,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:24,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:29,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:29,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:29,603 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:30,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:35,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:39,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:14:40,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:44,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:46,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:51,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:56,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:59,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:59,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:59,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:59,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_f15d4e86eefcf39e1eae.png with policy now +2023-07-12 10:14:59,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_8826c1292b54e84dc27b.png with policy now +2023-07-12 10:14:59,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_02cad4f86332c4d30e0f.png with policy now +2023-07-12 10:14:59,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_b90423e76bb46ed0bd51.png with policy now +2023-07-12 10:14:59,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:14:59,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:14:59,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:14:59,895 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:00,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:15:00,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 10:15:00,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 10:15:00,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 10:15:00,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 10:15:00,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:15:00,140 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pakh2qdm-media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 10:15:00,150 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zwb74alu-media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 10:15:00,200 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ug0k2e8w-media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 10:15:00,207 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ih8ew1uf-media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 10:15:00,560 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/989qnze8-media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 10:15:00,777 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ufxvj7v-media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 10:15:00,900 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhkwbe7n-media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 10:15:01,135 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ihzn3xgn-media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 10:15:01,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:06,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:09,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:09,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:15:10,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:11,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:14,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:14,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:14,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:15:17,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:19,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:19,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:19,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:19,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:20,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:15:20,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:23,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:28,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:29,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:29,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:29,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:15:30,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:30,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:30,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:30,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:30,254 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:33,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:38,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:39,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:15:39,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:39,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:39,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:40,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:40,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:15:40,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:43,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:44,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:15:48,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:50,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:50,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:50,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:50,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:50,340 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:54,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:59,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:59,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:59,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:59,999 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:00,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:00,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:00,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:00,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:00,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:00,640 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:05,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:09,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:16:10,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:10,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:10,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:10,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:10,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:10,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:10,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:15,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:16:15,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:16:15,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:15,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:20,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:20,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:20,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:20,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:20,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:20,632 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:21,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:26,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:30,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:16:30,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:16:30,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:30,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:30,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:30,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:30,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:30,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:30,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:31,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:35,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:35,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:35,303 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:35,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:36,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:39,907 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:16:40,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:40,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:41,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:45,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:16:45,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:16:45,288 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:47,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:52,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:57,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:00,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:00,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:00,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:02,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:07,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:09,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:17:12,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:15,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:15,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:15,529 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:18,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:23,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:28,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:29,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,377 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_f4dd09917aefe94149cf.png with policy now +2023-07-12 10:17:29,438 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_807425807eddcc0e0aaa.png with policy now +2023-07-12 10:17:29,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_30096936a6475e462a39.png with policy now +2023-07-12 10:17:29,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_8fe94ed8c310dae4cd3c.png with policy now +2023-07-12 10:17:29,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:29,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:17:29,502 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:17:29,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:17:29,745 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/87puz1sm-media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 10:17:29,817 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xbey8ss3-media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 10:17:29,846 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gkt6ivub-media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 10:17:29,855 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mp4bphr3-media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 10:17:30,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:30,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:30,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:17:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 10:17:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 10:17:30,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 10:17:30,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 10:17:30,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:17:31,068 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3vxidlgg-media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 10:17:31,080 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u09nhyhp-media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 10:17:31,084 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z1f7zqo9-media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 10:17:31,085 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cpl48eo2-media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 10:17:33,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:39,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:39,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:39,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:17:40,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:17:44,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:46,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:46,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:46,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:48,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:48,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:17:48,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:17:48,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:17:49,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:50,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:17:50,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:17:55,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:59,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:59,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:17:59,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:17:59,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:00,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:00,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:01,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:01,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:01,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:01,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:06,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:09,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:09,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:09,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:09,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:09,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:18:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:11,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:16,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:16,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:16,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:17,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:19,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:19,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:19,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:19,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:20,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:20,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:23,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:28,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:30,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:30,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:30,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:30,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:31,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:31,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:31,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:31,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:31,912 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:33,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:39,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:39,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:39,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:39,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:39,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:39,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:18:41,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:41,076 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:44,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:47,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:47,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:47,051 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:49,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:49,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:49,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:49,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:50,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:51,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:51,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:55,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:59,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:59,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:59,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:59,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:19:00,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:01,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:19:01,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:19:02,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:02,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:04,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:19:04,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:19:04,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:19:04,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:19:05,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:09,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:19:10,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:11,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:19:11,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:19:15,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:17,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:19:17,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:17,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:21,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:26,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:31,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:32,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:19:32,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:32,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:36,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:39,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:19:41,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:46,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:47,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:19:47,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:47,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:52,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:57,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:58,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_d5445284771d7b2c66a0.png with policy now +2023-07-12 10:19:58,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_c867e6da32e13d963e59.png with policy now +2023-07-12 10:19:58,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_815ded56ecd664726052.png with policy now +2023-07-12 10:19:58,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,717 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_e5fd4b41b8c0c33c93ac.png with policy now +2023-07-12 10:19:58,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:19:58,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:19:58,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:19:58,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:19:58,972 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d91som57-media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 10:19:59,005 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z4s5lui1-media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 10:19:59,041 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xd0ug1by-media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 10:19:59,124 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jqtvn2pt-media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 10:20:01,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:01,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 10:20:01,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 10:20:01,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 10:20:01,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 10:20:01,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:20:01,773 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vmpfkiwa-media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 10:20:01,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pnjz9juk-media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 10:20:01,783 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/skyxn26h-media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 10:20:01,797 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qum8izu1-media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 10:20:02,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:02,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:02,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:03,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:08,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:08,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:09,915 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:20:11,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:14,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:17,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:17,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:17,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:18,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:18,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:18,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:18,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:20,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:21,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:21,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:26,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:27,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:27,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:27,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:27,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:31,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:31,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:32,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:32,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:32,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:32,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:37,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:37,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:37,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:37,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:37,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:39,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:20:41,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:41,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:42,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:47,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:47,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:47,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:47,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:48,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:48,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:48,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:48,480 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:51,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:51,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:53,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:58,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:58,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:58,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:58,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:58,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:02,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:21:02,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:02,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:03,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:09,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:09,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:21:10,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:10,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:10,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:10,135 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:14,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:18,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:21:18,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:18,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:20,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:20,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:20,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:20,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:20,701 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:21,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:21,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:25,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:30,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:30,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:30,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:30,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:30,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:31,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:31,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:33,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:33,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:35,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:35,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:35,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:35,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:36,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:39,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:21:41,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:41,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:41,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:46,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:48,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:21:48,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:48,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:52,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:57,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:02,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:03,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:03,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:03,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:08,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:09,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:22:13,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:18,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:18,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:18,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:19,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:24,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:29,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:29,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_edd32e44eaf81373d115.png with policy now +2023-07-12 10:22:29,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_db2b2e82dd30d35d1311.png with policy now +2023-07-12 10:22:29,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_8dcd67f2db2d9902e40b.png with policy now +2023-07-12 10:22:29,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,732 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_97ab1f90f62f7fd81135.png with policy now +2023-07-12 10:22:29,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:29,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:22:29,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:22:29,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:22:30,155 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8baaxgt7-media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 10:22:30,164 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9414n9xa-media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 10:22:30,177 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bwktdb8q-media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 10:22:30,209 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gmw3sgmp-media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 10:22:32,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:22:32,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 10:22:32,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 10:22:32,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 10:22:32,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 10:22:32,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:22:32,530 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g9tz651v-media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 10:22:32,532 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wvt5wsen-media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 10:22:32,548 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2roygn9-media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 10:22:32,671 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xiexvg7o-media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 10:22:33,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:33,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:33,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:35,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:39,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:22:40,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:40,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:42,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:22:46,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:48,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:50,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:50,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:22:50,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:22:50,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:22:52,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:52,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:22:52,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:22:57,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:58,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:58,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:22:58,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:22:58,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:02,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:02,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:03,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:03,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:03,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:03,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:08,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:09,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:09,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:09,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:09,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:09,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:23:12,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:12,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:13,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:18,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:18,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:18,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:18,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:19,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:19,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:19,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:19,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:23,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:29,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:29,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:29,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:29,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:29,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:32,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:32,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:34,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:34,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:34,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:34,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:39,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:23:39,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:40,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:40,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:40,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:40,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:42,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:42,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:45,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:51,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:51,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:51,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:51,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:51,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:51,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:51,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:51,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:52,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:52,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:56,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:01,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:24:01,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:01,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:24:01,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:24:01,731 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:24:02,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:24:02,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:24:06,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:06,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:06,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:06,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:24:06,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:24:06,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:24:06,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:24:06,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:09,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:24:11,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:12,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:24:12,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:24:16,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:21,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:21,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:21,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:22,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:27,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:32,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:36,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:36,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:36,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:37,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:39,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:24:42,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:47,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:51,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:51,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:51,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:53,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:58,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:00,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,605 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_b53f3dded240b500b1cb.png with policy now +2023-07-12 10:25:00,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_a061712ad948dbaccbd5.png with policy now +2023-07-12 10:25:00,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_129f224b0015451a7862.png with policy now +2023-07-12 10:25:00,634 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,634 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_6c96769927607cfa02a7.png with policy now +2023-07-12 10:25:00,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:00,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:00,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:00,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:00,907 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ogauxbch-media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 10:25:00,935 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6f6z8bee-media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 10:25:00,942 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02rrpi4j-media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 10:25:00,956 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8o2y4t2x-media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 10:25:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 10:25:02,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 10:25:02,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 10:25:02,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 10:25:02,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:25:03,195 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c8qnj3dd-media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 10:25:03,209 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lkzltoas-media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 10:25:03,249 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kajg4clw-media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 10:25:03,296 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tmg94lwf-media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 10:25:03,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:06,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:06,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:06,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:09,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:09,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:25:10,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:12,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:14,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:19,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:19,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:19,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:19,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:20,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:21,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:21,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:21,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:22,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:22,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:26,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:29,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:29,708 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:29,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:29,711 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:31,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:32,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:32,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:36,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:36,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:36,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:38,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:39,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:39,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:39,448 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:39,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:39,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:25:42,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:42,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:44,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:49,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:49,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:49,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:49,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:49,697 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:51,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:51,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:51,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:52,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:52,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:54,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:00,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:00,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:00,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:00,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:00,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:03,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:03,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:05,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:06,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:06,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:09,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:26:10,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:11,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:11,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:11,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:11,116 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:13,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:13,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:16,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:21,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:21,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:21,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:21,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:21,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:21,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:21,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:23,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:23,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:26,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:31,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:31,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:31,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:31,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:31,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:33,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:33,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:36,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:36,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:26:36,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:36,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:36,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:36,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:36,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:36,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:39,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:26:41,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:43,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:43,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:46,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:51,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:26:51,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:51,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:52,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:57,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:02,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:06,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:06,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:06,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:07,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:09,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:27:12,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:17,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:21,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:21,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:21,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:23,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:28,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:30,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:30,985 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_e2238e57e86ad9eb1236.png with policy now +2023-07-12 10:27:31,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:31,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_fbe171c1140c25cd1cb2.png with policy now +2023-07-12 10:27:31,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:31,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_eb52c1f3852b2db713d6.png with policy now +2023-07-12 10:27:31,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:31,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_fb4911582a74ff6e59f1.png with policy now +2023-07-12 10:27:31,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:31,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:27:31,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:27:31,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:27:31,319 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5an5exb6-media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 10:27:31,398 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27hpj0zf-media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 10:27:31,403 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rxflr06c-media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 10:27:31,644 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pzzoypxh-media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 10:27:33,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:27:33,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 10:27:33,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 10:27:33,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 10:27:33,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 10:27:33,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:27:34,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:34,724 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8duytq7b-media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 10:27:34,854 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yroi7zq6-media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 10:27:35,107 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gffecb8v-media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 10:27:35,281 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nyszv03s-media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 10:27:36,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:36,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:36,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:39,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:27:39,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:41,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:43,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:27:45,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:50,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:50,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:27:50,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:27:50,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:27:51,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:52,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:52,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:52,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:54,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:27:54,552 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:27:57,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:59,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:59,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:27:59,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:27:59,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:02,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:05,567 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:05,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:07,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:07,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:07,669 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:07,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:09,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:28:10,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:10,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:10,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:10,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:13,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:16,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:16,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:19,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:20,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:20,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:20,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:20,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:23,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:23,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:23,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:24,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:27,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:27,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:30,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:31,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:31,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:31,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:31,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:36,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:38,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:38,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:38,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:38,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:38,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:39,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:28:41,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:41,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:41,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:41,677 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:41,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:46,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:49,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:49,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:51,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:51,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:51,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:51,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:51,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:53,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:53,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:53,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:57,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:00,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:29:00,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:29:01,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:29:01,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:29:01,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:29:01,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:29:02,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:06,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:29:06,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:29:06,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:29:06,393 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:29:08,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:08,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:08,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:08,912 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:09,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:29:11,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:29:11,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:29:13,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:18,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:23,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:24,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:24,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:24,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:29,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:34,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:39,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:39,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:39,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:39,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:29:39,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:44,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:49,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:54,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:54,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:54,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:55,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:00,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:01,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_1a8029b03d2f4e8ba6b5.png with policy now +2023-07-12 10:30:01,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_a3765156a53105d7a44f.png with policy now +2023-07-12 10:30:01,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_7b6d1552c5731ce4d99a.png with policy now +2023-07-12 10:30:01,197 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,197 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_45f174c3353c67e7d2fa.png with policy now +2023-07-12 10:30:01,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:01,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:01,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:01,241 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:01,428 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tuyutaup-media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 10:30:01,434 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/httc3xuk-media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 10:30:01,454 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbvtqk5b-media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 10:30:01,523 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpg6z62d-media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 10:30:06,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:07,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:07,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:07,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 10:30:07,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 10:30:07,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 10:30:07,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 10:30:07,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:30:07,607 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uniypzz0-media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 10:30:07,614 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/geip8mmw-media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 10:30:07,618 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bfa3yq8w-media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 10:30:07,619 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyeq0c04-media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 10:30:09,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:30:09,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:09,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:09,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:30:11,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:11,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:17,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:18,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:20,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:20,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:20,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:20,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:22,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:24,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:30:24,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:24,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:27,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:29,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:29,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:30,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:30,682 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:30,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:30,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:33,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:38,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:39,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:39,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:30:39,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:39,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:30:40,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:40,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:41,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:41,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:41,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:41,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:44,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:49,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:51,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:51,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:52,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:52,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:52,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:52,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:54,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:54,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:55,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:00,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:02,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:02,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:02,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:02,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:02,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:02,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:06,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:09,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:09,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:31:11,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:12,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:12,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:12,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:12,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:13,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:13,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:16,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:22,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:22,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:22,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:22,698 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:22,700 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:24,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:24,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:24,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:24,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:24,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:27,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:32,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:32,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:32,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:32,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:32,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:35,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:35,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:37,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:37,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:37,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:37,416 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:37,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:39,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:31:39,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:39,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:39,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:43,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:46,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:46,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:48,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:53,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:55,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:58,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:03,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:08,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:09,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:32:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:32:10,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:14,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:19,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:24,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:25,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:25,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:29,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:31,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_f73a3d21c0ad929cc0df.png with policy now +2023-07-12 10:32:31,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,702 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png with policy now +2023-07-12 10:32:31,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_2345ba38c318660ea20d.png with policy now +2023-07-12 10:32:31,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_4acac46bf9cdc8077cd8.png with policy now +2023-07-12 10:32:31,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:32:31,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:32:31,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:32:31,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:32:32,049 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpb15br1-media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 10:32:32,068 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdnyah1p-media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 10:32:32,079 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/msq3ibcj-media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 10:32:32,135 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f5ws0o1d-media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 10:32:34,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:39,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:32:39,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:40,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:32:40,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:40,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:41,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:32:41,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:32:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:32:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 10:32:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 10:32:41,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 10:32:41,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 10:32:41,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:32:42,461 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z2g9q0ur-media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 10:32:42,510 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x0poqlfz-media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 10:32:42,539 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ws74ovw8-media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 10:32:42,564 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpd2fk6g-media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 10:32:45,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:50,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:50,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:32:50,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:32:50,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:32:50,948 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:32:52,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:32:52,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:32:55,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:32:55,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:55,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:55,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:01,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:01,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:01,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:01,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:01,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:03,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:03,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:06,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:09,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:33:10,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:10,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:10,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:10,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:10,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:10,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:10,898 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:11,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:14,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:14,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:16,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:20,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:20,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:20,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:20,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:22,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:25,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:25,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:25,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:26,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:26,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:28,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:31,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:31,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:31,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:31,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:33,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:38,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:39,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:33:40,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:40,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:40,835 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:41,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:41,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:41,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:41,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:44,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:48,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:48,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:49,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:52,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:52,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:52,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:52,750 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:54,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:55,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:55,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:55,835 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:59,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:59,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:59,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:02,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:34:02,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:34:02,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:34:02,820 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:34:04,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:07,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:34:07,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:34:07,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:34:07,186 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:34:09,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:34:09,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:10,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:34:10,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:34:10,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:10,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:10,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:15,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:20,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:25,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:25,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:25,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:25,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:31,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:36,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:39,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:34:41,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:41,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:41,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:41,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:46,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:51,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:56,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:56,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:56,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:56,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:01,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:01,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_ce5f5d922b12cb9a8e58.png with policy now +2023-07-12 10:35:01,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_55683596f533718ecbc7.png with policy now +2023-07-12 10:35:01,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_9cd53a6492dff1119fcd.png with policy now +2023-07-12 10:35:01,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_4e5b7eac0474303b7db6.png with policy now +2023-07-12 10:35:01,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:01,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:01,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:01,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:01,694 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/45b1v69l-media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 10:35:01,714 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ver8pl49-media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 10:35:01,813 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7cwq2ety-media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 10:35:01,829 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vshm5jod-media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 10:35:05,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:05,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 10:35:05,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 10:35:05,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 10:35:05,598 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 10:35:05,598 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:35:05,975 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/105jvs7p-media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 10:35:05,977 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uh6m09pp-media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 10:35:05,978 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5ktjowrr-media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 10:35:05,981 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/etlcnkv7-media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 10:35:06,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:09,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:35:11,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:11,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:11,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:35:11,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:11,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:16,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:17,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:20,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:20,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:20,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:20,609 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:22,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:26,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:26,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:26,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:35:27,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:27,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:28,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:31,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:31,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:31,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:31,015 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:33,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:38,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:38,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:38,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:39,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:35:40,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:41,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:41,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:41,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:41,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:41,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:41,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:35:44,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:49,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:49,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:49,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:51,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:51,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:51,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:51,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:54,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:56,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:56,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:56,334 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:00,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:00,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:00,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:01,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:01,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:01,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:01,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:05,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:09,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:36:10,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:11,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:11,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:11,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:11,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:11,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:12,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:12,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:12,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:12,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:16,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:22,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:22,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:22,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:22,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:22,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:22,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:22,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:26,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:26,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:26,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:28,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:31,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:31,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:31,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:31,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:33,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:36,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:36,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:36,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:36,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:39,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:39,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:36:41,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:41,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:41,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:44,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:44,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:44,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:49,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:54,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:56,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:56,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:56,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:59,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:04,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:09,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:37:10,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:11,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:11,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:11,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:16,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:21,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:26,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:26,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:30,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_a5bdcb2b01b23f824fcf.png with policy now +2023-07-12 10:37:30,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_e419bc8806d8bdc57fa0.png with policy now +2023-07-12 10:37:30,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,918 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_48e39b87d59cc06724b3.png with policy now +2023-07-12 10:37:30,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_4436a3e5cc352d2a1017.png with policy now +2023-07-12 10:37:30,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:30,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:37:30,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:37:30,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:37:31,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bdkiyd14-media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 10:37:31,604 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpqsbzbg-media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 10:37:31,752 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eni5dlxq-media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 10:37:31,797 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5p88qyx4-media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 10:37:31,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:37,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:39,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:37:40,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:37:40,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:37:40,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 10:37:40,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 10:37:40,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 10:37:40,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 10:37:40,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:37:40,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:40,763 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2mk7nq4t-media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 10:37:40,771 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ktv84ge-media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 10:37:40,774 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xl7q67at-media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 10:37:40,788 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8e2r2crs-media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 10:37:42,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:42,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:42,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:42,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:48,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:50,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:50,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:37:50,676 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:37:50,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:37:51,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:37:51,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:37:53,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:57,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:57,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:57,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:59,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:59,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:59,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:37:59,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:37:59,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:02,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:02,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:04,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:09,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:09,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:09,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:09,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:09,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:38:09,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:12,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:12,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:12,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:13,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:13,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:15,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:19,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:19,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:19,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:19,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:20,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:24,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:24,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:26,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:27,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:27,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:27,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:31,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:31,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:31,047 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:31,049 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:32,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:35,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:35,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:37,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:39,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:38:42,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:42,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:42,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:42,100 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:42,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:42,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:42,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:43,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:46,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:46,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:49,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:52,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:52,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:52,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:52,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:54,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:57,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:57,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:57,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:57,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:57,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:59,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:02,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:39:02,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:39:02,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:39:02,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:39:05,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:07,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:39:07,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:39:07,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:39:07,484 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:39:08,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:39:08,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:39:09,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:39:10,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:12,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:12,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:39:16,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:19,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:39:21,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:26,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:27,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:39:31,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:36,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:39,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:39:41,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:42,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:42,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:42,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:39:47,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:52,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:57,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:57,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:57,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:57,851 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:01,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_03d4142985e8dc5f61fc.png with policy now +2023-07-12 10:40:01,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,684 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_13c9cc17dc94ed86351c.png with policy now +2023-07-12 10:40:01,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_97a6019e1743b1256999.png with policy now +2023-07-12 10:40:01,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,783 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_1248be5f76e40ac7db9c.png with policy now +2023-07-12 10:40:01,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:01,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:01,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:01,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:02,040 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpbc2rn0-media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 10:40:02,041 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e121qmzs-media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 10:40:02,143 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9yttxo7a-media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 10:40:02,165 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vlw43936-media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 10:40:02,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:03,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:03,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 10:40:03,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 10:40:03,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 10:40:03,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 10:40:03,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:40:04,331 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uyi9il8b-media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 10:40:04,334 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6e9akvce-media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 10:40:04,346 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c55ru7mk-media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 10:40:04,420 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tm4v2t8t-media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 10:40:07,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:09,959 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:40:11,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:13,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:13,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:13,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:13,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:14,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:18,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:20,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:20,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:20,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:20,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:24,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:26,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:26,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:28,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:28,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:28,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:29,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:30,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:30,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:30,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:30,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:34,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:37,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:39,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:40:39,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:40,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:40,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:40,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:40,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:43,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:43,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:43,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:45,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:48,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:48,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:50,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:50,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:50,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:50,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:50,947 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:55,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:58,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:58,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:58,487 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:59,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:59,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:01,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:01,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:01,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:01,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:01,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:06,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:09,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:41:10,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:10,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:11,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:11,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:11,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:11,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:12,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:13,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:13,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:13,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:41:17,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:21,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:21,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:21,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:21,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:21,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:21,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:22,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:28,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:28,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:28,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:28,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:41:31,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:31,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:31,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:31,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:32,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:32,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:33,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:36,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:36,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:36,542 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:36,544 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:39,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:39,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:41:43,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:43,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:43,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:43,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:43,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:41:44,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:49,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:58,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:58,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:58,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:00,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:05,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:09,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:42:10,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:13,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:13,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:13,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:16,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:21,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:26,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:28,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:31,031 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_8cdd64e4abd06459cc31.png with policy now +2023-07-12 10:42:31,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_5f9749e51f806c3a7042.png with policy now +2023-07-12 10:42:31,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,051 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_b1d38d2764a5721d15ac.png with policy now +2023-07-12 10:42:31,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_7e913201741ac8370fe9.png with policy now +2023-07-12 10:42:31,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:42:31,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:42:31,101 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:42:31,103 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:42:31,378 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/di9k3b4f-media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 10:42:31,399 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dv4bydl3-media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 10:42:31,404 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w30qts4a-media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 10:42:31,412 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w22ha3ka-media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 10:42:32,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:37,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:38,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:42:38,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:42:38,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 10:42:38,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 10:42:38,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 10:42:38,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 10:42:38,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:42:39,011 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/upxvzl7t-media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 10:42:39,019 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/glfaly8p-media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 10:42:39,039 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rftl9y8f-media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 10:42:39,041 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ul283g29-media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 10:42:39,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:42:42,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:42:43,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:43,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:43,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:43,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:48,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:49,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:42:51,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:42:51,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:42:51,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:42:51,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:42:54,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:58,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:58,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:58,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:59,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:00,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:00,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:01,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:01,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:01,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:01,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:05,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:09,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:43:10,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:11,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:11,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:11,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:11,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:11,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:11,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:13,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:43:13,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:43:13,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:43:16,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:21,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:21,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:21,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:21,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:21,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:26,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:29,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:43:29,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:43:29,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:43:32,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:32,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:32,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:32,136 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:32,138 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:33,751 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:33,751 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:37,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:39,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:43:42,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:42,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:42,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:42,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:43,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:44,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:43:44,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:43:44,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:43:44,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:44,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:48,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:52,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:52,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:52,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:52,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:53,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:56,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:56,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:59,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:00,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:00,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:00,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:02,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:44:02,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:44:02,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:44:02,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:44:04,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:07,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:44:07,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:44:08,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:44:08,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:44:08,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:44:08,075 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:44:09,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:44:09,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:14,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:15,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:15,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:15,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:18,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:44:18,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:44:20,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:25,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:30,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:30,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:30,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:30,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:35,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:39,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:44:40,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:45,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:45,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:45,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:46,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:51,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:56,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:00,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:00,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:00,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:02,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:02,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,692 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_35fe8e67bdbd375bad51.png with policy now +2023-07-12 10:45:02,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,702 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_e4060fd5427ae0ecee26.png with policy now +2023-07-12 10:45:02,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,711 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_8ef00575ef619617eb8f.png with policy now +2023-07-12 10:45:02,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_8c7622cbbe1b8f2266c2.png with policy now +2023-07-12 10:45:02,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:02,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:02,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:02,769 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:03,004 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tn65ea73-media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 10:45:03,015 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4deabl7-media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 10:45:03,032 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ekhjs1ru-media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 10:45:03,110 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s8rmp1uv-media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 10:45:07,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:09,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:45:12,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:12,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:13,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:13,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:13,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 10:45:13,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 10:45:13,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 10:45:13,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 10:45:13,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:45:13,696 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1zuourkf-media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 10:45:13,710 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8jbdzebp-media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 10:45:13,710 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xbrod0rb-media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 10:45:13,711 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0a98pobp-media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 10:45:15,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:15,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:15,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:18,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:22,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:22,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:22,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:22,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:23,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:24,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:24,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:28,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:30,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:30,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:30,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:31,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:31,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:31,923 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:31,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:33,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:35,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:35,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:39,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:39,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:45:41,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:41,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:41,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:41,842 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:44,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:46,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:46,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:46,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:46,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:46,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:50,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:51,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:51,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:51,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:51,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:55,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:57,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:57,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:00,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:01,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:01,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:01,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:02,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:02,179 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:02,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:05,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:08,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:08,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:09,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:46:11,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:13,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:13,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:13,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:13,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:16,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:16,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:16,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:17,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:19,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:19,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:23,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:24,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:24,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:24,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:24,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:29,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:30,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:30,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:31,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:31,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:31,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:34,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:34,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:34,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:34,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:34,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:39,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:39,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:39,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:39,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:39,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:39,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:46:41,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:41,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:44,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:46,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:46,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:46,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:50,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:55,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:00,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:01,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:01,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:01,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:06,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:09,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:47:11,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:16,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:16,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:16,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:17,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:22,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:27,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:31,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:31,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:31,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:33,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:47:36,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 10:47:36,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 10:47:36,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 10:47:36,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 10:47:37,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:47:37,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:38,339 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vrx9tm2v-media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 10:47:40,179 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/80lv8ia6-media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 10:47:41,053 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/briwoxw9-media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 10:47:41,057 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o5w4tttj-media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 10:47:41,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:41,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_5dcd5f49c5610ed98e57.png with policy now +2023-07-12 10:47:41,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_ceacc8064824cf24b9c3.png with policy now +2023-07-12 10:47:41,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_13fad504fb5c2ac0ee04.png with policy now +2023-07-12 10:47:41,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_0287eb8f3f80591bc234.png with policy now +2023-07-12 10:47:41,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:47:41,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:47:41,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:47:41,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:47:42,106 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7lp5breu-media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 10:47:42,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:47:42,602 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pnrbtnoq-media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 10:47:42,751 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jx8k58q2-media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 10:47:42,785 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b3pzveww-media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 10:47:46,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:47,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:47,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:47,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:48,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:47:48,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:47:51,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:47:51,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:47:51,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:47:51,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:47:52,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:58,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:59,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:47:59,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:02,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:02,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:02,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:02,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:02,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:02,154 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:03,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:08,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:09,976 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:48:10,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:10,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:12,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:12,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:12,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:12,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:14,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:17,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:17,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:17,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:20,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:21,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:21,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:23,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:23,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:23,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:23,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:25,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:30,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:32,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:32,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:32,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:32,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:32,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:33,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:33,376 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:33,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:33,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:36,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:39,977 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:48:41,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:43,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:43,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:43,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:43,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:43,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:43,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:46,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:47,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:47,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:47,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:51,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:53,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:53,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:53,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:53,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:54,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:54,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:57,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:02,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:03,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:03,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:03,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:04,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:49:04,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:49:04,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:49:04,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:49:05,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:49:05,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:49:08,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:09,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:49:09,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:49:09,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:49:09,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:49:09,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:49:13,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:16,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:49:16,384 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:49:18,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:18,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:18,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:19,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:24,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:29,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:33,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:33,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:33,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:34,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:39,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:39,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:49:44,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:48,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:48,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:48,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:50,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:55,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:00,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:03,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:03,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:03,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:03,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,647 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_784615f77a2b4e7047a9.png with policy now +2023-07-12 10:50:03,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_38ab10654e678fd3808d.png with policy now +2023-07-12 10:50:03,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_0362f6603354bb565423.png with policy now +2023-07-12 10:50:03,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,650 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_12a798eb5ac3d54ffad6.png with policy now +2023-07-12 10:50:03,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:03,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:03,677 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:03,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:04,045 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dbupzzwk-media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 10:50:04,047 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9mkqpys6-media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 10:50:04,054 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/djqjry0m-media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 10:50:04,085 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bkz80ns6-media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 10:50:05,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:09,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:50:10,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:11,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:11,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 10:50:11,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:50:12,105 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uja8oyud-media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 10:50:12,111 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/osh04iah-media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 10:50:12,113 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dtfi4ck0-media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 10:50:12,134 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hgjn5niz-media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 10:50:13,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:16,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:19,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:19,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:19,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:22,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:22,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:22,596 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:22,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:22,598 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:22,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:22,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:28,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:32,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:32,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:32,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:32,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:33,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:33,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:34,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:34,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:34,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:34,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:39,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:39,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:50:42,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:42,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:42,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:42,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:44,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:44,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:44,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:49,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:49,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:49,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:49,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:52,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:52,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:52,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:52,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:55,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:55,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:55,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:00,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:03,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:03,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:03,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:03,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:05,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:05,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:05,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:05,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:06,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:06,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:09,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:51:11,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:12,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:12,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:12,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:12,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:16,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:18,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:18,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:20,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:20,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:20,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:21,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:23,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:23,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:23,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:23,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:27,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:29,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:29,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:32,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:34,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:34,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:34,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:34,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:35,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:35,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:35,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:38,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:38,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:38,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:38,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:38,459 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:39,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:51:40,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:40,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:43,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:48,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:50,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:54,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:59,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:04,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:05,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:05,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:05,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:09,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:09,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:52:14,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:19,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:20,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:20,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:20,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:25,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:30,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:32,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_62ad7e090c150c4642cd.png with policy now +2023-07-12 10:52:32,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_2342de4c23f89a71e70a.png with policy now +2023-07-12 10:52:32,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_12a6034cf8cfc23b5df1.png with policy now +2023-07-12 10:52:32,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,816 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_59ef69c8ce6d4ab2107c.png with policy now +2023-07-12 10:52:32,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:52:32,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:52:32,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:52:32,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:52:33,161 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lhct4zfo-media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 10:52:33,174 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/er358v07-media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 10:52:33,174 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m5gy7hv6-media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 10:52:33,189 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1wqs60gw-media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 10:52:35,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:52:35,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 10:52:35,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 10:52:35,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 10:52:35,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 10:52:35,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:52:35,783 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4fgnx7l8-media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 10:52:35,806 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ty3d3z8u-media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 10:52:35,808 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/re2yimyp-media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 10:52:35,814 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/crft1tvd-media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 10:52:35,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:35,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:35,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:35,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:39,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:52:41,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:43,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:52:46,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:52:47,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:50,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:52,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:52:52,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:52,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:52:52,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:52:52,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:52:57,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:52:57,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:52:58,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:02,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:02,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:02,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:02,248 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:04,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:06,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:06,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:06,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:08,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:08,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:09,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:09,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:53:12,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:12,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:12,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:12,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:14,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:19,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:19,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:19,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:21,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:21,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:21,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:22,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:22,487 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:22,487 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:22,489 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:24,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:30,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:30,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:30,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:32,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:32,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:32,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:32,655 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:35,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:37,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:37,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:37,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:39,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:53:40,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:41,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:41,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:42,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:42,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:42,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:42,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:45,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:50,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:52,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:52,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:52,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:54,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:54,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:54,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:54,491 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:56,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:02,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:03,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:54:03,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:54:04,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:54:04,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:54:04,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:54:04,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:54:07,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:07,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:07,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:54:07,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:09,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:54:09,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:54:09,468 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:54:09,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:54:09,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:54:13,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:14,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:54:14,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:54:18,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:22,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:22,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:54:22,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:23,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:28,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:33,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:37,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:37,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:54:37,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:38,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:39,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:54:43,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:48,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:52,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:52,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:54,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:59,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:03,900 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_404750014d2070b4b00e.png with policy now +2023-07-12 10:55:03,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_e5c153d3981bac3820e6.png with policy now +2023-07-12 10:55:03,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_305e21c628b26bb7d55e.png with policy now +2023-07-12 10:55:03,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_5996b8fa55b566dccbed.png with policy now +2023-07-12 10:55:03,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:03,977 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:03,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:03,981 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:04,228 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mu0v1g0i-media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 10:55:04,243 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4mr4b4hj-media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 10:55:04,255 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q1bneioz-media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 10:55:04,264 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c5sglp9n-media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 10:55:04,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:07,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:07,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:09,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:55:09,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:10,088 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:10,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:10,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 10:55:10,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 10:55:10,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 10:55:10,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 10:55:10,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:55:10,473 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ukwhcbq-media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 10:55:10,474 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xhm1fc6m-media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 10:55:10,489 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbd2r1z1-media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 10:55:10,497 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/luk7igg9-media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 10:55:13,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:15,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:20,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:21,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:55:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:22,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:23,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:23,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:23,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:23,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:25,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:31,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:32,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:32,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:32,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:32,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:32,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:32,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:37,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:37,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:55:37,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:37,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:39,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:55:42,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:42,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:42,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:42,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:42,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:43,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:43,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:48,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:52,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:55:52,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:52,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:53,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:53,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:53,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:53,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:53,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:54,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:54,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:58,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:03,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:03,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:03,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:03,542 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:04,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:05,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:05,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:07,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:07,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:07,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:09,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:56:09,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:13,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:13,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:13,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:13,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:15,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:16,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:16,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:21,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:22,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:22,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:22,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:24,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:24,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:24,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:24,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:26,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:32,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:34,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:34,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:34,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:34,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:37,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:37,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:37,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:37,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:38,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:38,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:39,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:39,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:39,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:39,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:39,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:56:43,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:48,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:49,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:49,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:53,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:53,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:53,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:53,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:58,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:03,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:08,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:08,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:08,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:08,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:09,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:57:14,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:19,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:23,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:23,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:23,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:24,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:29,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:33,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_a3c7e4b8847949aa758f.png with policy now +2023-07-12 10:57:33,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_d8ef17b50203820bbbe3.png with policy now +2023-07-12 10:57:33,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_5b8b7b248eeeadd01dc0.png with policy now +2023-07-12 10:57:33,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_3c27c596a729886e6078.png with policy now +2023-07-12 10:57:33,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:57:33,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:57:33,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:57:33,873 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:57:34,141 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2yrlbyhu-media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 10:57:34,144 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3e1zpirm-media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 10:57:34,144 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vmzmq9m4-media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 10:57:34,163 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wigy0pwg-media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 10:57:34,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:38,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:38,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:38,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:40,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:57:40,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:43,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:57:44,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:57:44,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:57:44,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 10:57:44,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 10:57:44,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 10:57:44,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 10:57:44,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:57:45,184 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dnfo82zr-media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 10:57:45,196 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/avg62cxt-media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 10:57:45,224 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h3tq612a-media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 10:57:45,231 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nmp4mlm6-media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 10:57:45,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:51,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:52,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:57:52,942 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:57:52,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:57:52,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:57:55,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:55,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:55,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:55,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:57:55,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:57:56,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:02,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:03,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:03,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:03,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:03,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:06,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:06,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:07,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:10,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:58:10,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:10,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:10,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:13,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:14,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:14,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:14,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:14,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:17,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:17,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:19,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:24,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:24,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:24,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:24,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:24,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:25,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:25,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:25,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:28,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:28,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:30,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:34,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:34,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:34,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:34,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:35,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:39,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:39,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:40,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:58:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:40,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:41,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:44,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:44,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:44,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:44,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:46,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:51,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:51,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:52,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:55,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:55,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:55,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:56,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:56,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:56,101 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:56,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:58,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:02,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:59:02,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:59:03,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:06,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:59:06,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:59:06,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:59:06,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:59:08,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:10,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:59:10,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:10,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:10,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:59:10,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:59:10,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:59:10,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:59:10,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:59:13,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:59:13,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:59:14,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:19,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:24,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:25,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:25,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:25,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:59:29,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:34,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:39,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:40,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:59:40,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:40,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:40,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:59:44,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:49,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:54,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:55,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:55,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:55,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:00,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:05,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_8d036206f070328af8f6.png with policy now +2023-07-12 11:00:05,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,104 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_b1cd6dc23fe6b08880be.png with policy now +2023-07-12 11:00:05,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,114 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_792f17b1a307463180d2.png with policy now +2023-07-12 11:00:05,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_87526415e87acfa56b82.png with policy now +2023-07-12 11:00:05,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:05,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:05,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:05,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:05,468 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bk2ur7nb-media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 11:00:05,490 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhk4va1v-media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 11:00:05,507 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l4825rsf-media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 11:00:05,533 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbrjrit0-media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 11:00:06,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:08,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:08,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 11:00:08,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 11:00:08,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 11:00:08,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 11:00:08,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:00:08,819 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wj7r3fds-media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 11:00:08,849 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8te0cvf-media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 11:00:08,852 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j5r73jxf-media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 11:00:08,853 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gbtyx5vj-media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 11:00:10,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:00:10,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:10,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:10,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:11,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:14,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:16,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:19,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:22,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:24,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:24,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:24,413 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:24,415 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:25,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:25,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:25,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:28,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:30,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:33,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:34,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:34,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:34,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:34,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:38,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:40,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:00:41,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:41,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:41,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:41,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:41,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:44,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:45,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:45,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:45,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:45,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:50,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:52,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:52,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:55,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:55,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:55,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:55,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:55,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:57,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:57,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:57,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:01,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:03,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:03,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:06,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:06,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:06,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:06,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:06,480 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:10,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:01:11,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:13,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:13,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:13,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:14,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:14,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:16,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:16,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:16,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:16,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:17,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:22,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:25,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:25,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:27,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:27,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:27,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:27,243 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:28,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:29,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:29,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:29,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:33,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:36,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:36,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:37,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:37,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:37,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:37,176 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:39,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:40,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:01:41,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:41,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:41,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:41,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:44,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:44,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:44,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:44,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:47,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:47,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:49,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:54,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:59,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:59,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:59,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:00,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:05,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:10,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:02:11,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:14,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:14,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:14,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:16,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:21,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:26,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:29,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:29,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:29,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:32,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:36,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_af826553a0f3457c21a4.png with policy now +2023-07-12 11:02:36,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_1e363e4fe52a8dd1f3d5.png with policy now +2023-07-12 11:02:36,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_01eb7980170d3a100648.png with policy now +2023-07-12 11:02:36,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,298 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_f752872a939e31b6a407.png with policy now +2023-07-12 11:02:36,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:02:36,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:02:36,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:02:36,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:02:36,579 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c21mto1z-media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 11:02:36,589 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l41mzesa-media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 11:02:36,604 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/67uslcst-media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 11:02:36,605 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icjluwj2-media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 11:02:37,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:40,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:02:42,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:43,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:02:43,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:02:43,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 11:02:43,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 11:02:43,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 11:02:43,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 11:02:43,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:02:43,572 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/87sll2x1-media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 11:02:43,589 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/swopib3z-media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 11:02:43,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u2aev0gy-media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 11:02:43,603 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6pynq7oc-media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 11:02:44,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:44,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:44,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:46,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:02:48,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:53,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:54,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:02:55,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:02:55,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:02:55,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:02:55,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:02:59,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:59,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:59,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:59,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:04,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:05,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:05,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:05,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:05,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:09,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:10,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:03:14,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:14,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:14,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:03:15,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:15,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:15,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:15,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:15,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:16,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:16,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:20,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:25,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:27,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:27,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:27,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:27,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:27,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:27,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:29,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:29,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:29,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:03:31,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:36,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:37,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:37,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:37,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:37,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:38,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:38,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:40,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:03:42,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:44,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:03:47,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:47,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:47,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:47,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:47,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:49,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:49,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:53,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:56,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:56,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:56,999 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:57,001 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:59,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:59,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:59,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:59,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:00,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:04:00,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:04:04,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:08,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:04:08,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:04:08,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:04:08,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:04:10,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:04:10,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:11,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:04:11,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:04:12,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:04:12,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:04:12,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:04:12,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:04:14,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:14,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:14,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:15,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:20,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:22,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:04:22,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:04:25,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:29,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:29,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:29,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:31,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:36,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:40,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:04:42,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:44,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:44,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:47,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:52,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:57,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:59,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:59,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:59,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:02,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:07,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_cbe3a2e2923fea324bbe.png with policy now +2023-07-12 11:05:07,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_ece7f7211bafcdaa62b2.png with policy now +2023-07-12 11:05:07,503 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_8852be70b2d4d0b9cda6.png with policy now +2023-07-12 11:05:07,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_60a5b23b74be955f2c27.png with policy now +2023-07-12 11:05:07,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:07,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:07,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:07,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:07,762 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/69hiaa3o-media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 11:05:07,814 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r74jo1n6-media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 11:05:07,816 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0d5nrzrh-media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 11:05:07,818 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w2e4awfw-media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 11:05:08,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:10,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:05:14,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:15,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:05:15,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:05:15,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:17,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:17,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:17,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 11:05:17,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 11:05:17,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 11:05:17,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 11:05:17,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:05:18,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:19,751 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k8wj7adu-media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 11:05:20,026 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rpfzb5h5-media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 11:05:20,054 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vvgjajin-media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 11:05:20,273 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fjmmeyda-media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 11:05:20,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:25,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:27,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:27,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:27,781 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:27,783 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:28,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:28,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:30,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:05:30,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:05:30,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:31,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:36,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:37,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:37,915 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:37,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:37,918 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:40,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:40,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:40,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:05:42,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:45,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:05:45,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:05:45,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:47,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:47,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:47,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:47,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:47,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:51,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:51,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:52,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:58,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:58,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:58,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:58,603 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:58,605 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:00,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:00,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:00,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:02,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:02,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:03,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:08,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:08,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:08,437 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:08,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:08,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:10,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:06:13,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:13,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:14,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:15,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:15,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:15,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:18,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:18,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:18,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:18,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:20,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:24,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:24,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:26,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:29,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:29,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:29,034 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:29,036 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:31,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:31,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:31,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:31,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:35,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:35,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:37,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:38,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:38,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:38,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:38,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:40,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:06:43,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:43,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:43,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:43,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:43,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:46,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:46,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:46,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:46,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:46,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:48,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:53,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:58,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:01,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:01,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:01,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:04,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:09,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:10,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:07:15,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:16,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:16,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:16,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:20,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:25,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:30,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:31,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:31,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:31,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:35,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:37,273 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_e1afc2c90dd83c53afe8.png with policy now +2023-07-12 11:07:37,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_3a9f356aa83d5b8c3154.png with policy now +2023-07-12 11:07:37,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,300 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_feb61af945632b2e5a0d.png with policy now +2023-07-12 11:07:37,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,632 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_bd16c2dda6cb242546bd.png with policy now +2023-07-12 11:07:37,649 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p7otx2m5-media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 11:07:37,651 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mxm33j5k-media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 11:07:37,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:07:37,675 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:07:37,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:07:37,677 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:07:37,754 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8j82gxbp-media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 11:07:38,009 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdamez17-media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 11:07:40,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:07:40,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:41,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:07:41,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:07:41,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 11:07:41,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 11:07:41,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 11:07:41,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 11:07:41,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:07:42,026 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gjiwh2r6-media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 11:07:42,031 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sfwldm3q-media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 11:07:42,044 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8fse9ha6-media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 11:07:42,066 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hynv9ieb-media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 11:07:46,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:46,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:46,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:46,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:47,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:07:51,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:52,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:07:56,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:58,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:07:58,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:07:58,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:07:58,458 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:01,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:01,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:01,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:02,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:03,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:03,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:07,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:07,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:07,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:07,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:07,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:10,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:08:12,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:14,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:14,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:16,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:16,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:16,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:18,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:18,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:18,278 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:18,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:18,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:24,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:25,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:25,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:27,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:27,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:27,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:27,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:29,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:31,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:31,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:31,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:35,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:36,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:36,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:38,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:38,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:38,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:38,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:40,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:08:41,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:46,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:46,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:46,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:46,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:47,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:47,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:48,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:48,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:48,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:48,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:51,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:57,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:59,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:59,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:59,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:59,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:59,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:59,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:09:01,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:01,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:01,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:02,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:08,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:09,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:09:09,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:09:09,579 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:09:09,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:09:10,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:09:10,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:09:10,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:09:14,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:14,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:09:14,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:09:14,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:09:14,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:09:16,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:16,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:16,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:20,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:21,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:09:21,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:09:25,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:30,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:31,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:31,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:31,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:35,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:40,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:09:41,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:46,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:46,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:46,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:46,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:51,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:56,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:01,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:01,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:01,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:01,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:06,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:08,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,859 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_39f7775ce6273f5abdd3.png with policy now +2023-07-12 11:10:08,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_be22a91ba76075904920.png with policy now +2023-07-12 11:10:08,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,879 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_570101fe2f3838813987.png with policy now +2023-07-12 11:10:08,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_049933627fe17227a897.png with policy now +2023-07-12 11:10:08,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:08,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:08,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:08,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:09,187 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b3vkw3xu-media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 11:10:09,218 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0vqmt49s-media/images/validation_2660_049933627fe17227a897.png +2023-07-12 11:10:09,224 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f0bi2h2k-media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 11:10:09,244 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9e87psg-media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 11:10:10,033 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:10:12,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:16,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:10:16,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:16,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_049933627fe17227a897.png +2023-07-12 11:10:16,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 11:10:16,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 11:10:16,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 11:10:16,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:10:16,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:16,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:16,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:16,751 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2pqxqh68-media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 11:10:16,786 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wg92egh7-media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 11:10:16,792 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/52xtaraa-media/images/validation_2660_049933627fe17227a897.png +2023-07-12 11:10:16,840 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yjx44ah0-media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 11:10:17,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:19,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:23,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:27,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:28,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:28,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:28,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:28,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:28,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:31,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:31,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:31,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:34,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:38,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:10:38,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:39,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:39,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:39,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:39,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:40,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:10:40,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:45,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:46,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:46,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:46,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:49,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:49,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:49,118 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:49,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:49,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:10:49,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:51,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:56,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:00,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:00,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:00,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:00,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:00,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:00,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:02,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:02,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:02,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:02,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:07,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:10,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:11:10,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:10,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:10,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:10,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:11,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:11,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:13,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:17,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:17,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:17,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:18,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:20,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:20,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:20,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:20,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:22,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:22,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:24,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:30,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:30,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:30,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:30,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:30,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:33,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:33,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:33,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:33,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:33,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:35,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:40,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:11:41,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:41,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:41,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:41,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:41,377 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:44,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:44,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:46,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:46,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:46,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:46,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:46,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:48,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:48,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:48,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:51,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:56,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:01,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:03,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:03,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:03,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:06,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:10,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:12:12,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:17,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:18,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:18,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:18,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:22,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:27,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:32,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:33,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:33,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:33,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:37,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:40,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:12:40,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_b7c6a59e577b6aff9423.png with policy now +2023-07-12 11:12:40,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_8f7a128a9aebab91d68e.png with policy now +2023-07-12 11:12:40,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_abd6457914db5ecb11f5.png with policy now +2023-07-12 11:12:40,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_81b157e520543297899b.png with policy now +2023-07-12 11:12:40,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:12:40,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:12:40,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:12:40,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:12:40,907 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ky8wo4qf-media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 11:12:40,928 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdq4m6zn-media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 11:12:40,940 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgj989gd-media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 11:12:40,963 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z3yyocan-media/images/validation_2670_81b157e520543297899b.png +2023-07-12 11:12:42,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:47,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:48,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:48,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:48,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:50,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 11:12:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 11:12:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_81b157e520543297899b.png +2023-07-12 11:12:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:12:51,492 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8u2pi9kz-media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 11:12:51,524 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/09l1owfd-media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 11:12:51,553 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0dwyzo1e-media/images/validation_2670_81b157e520543297899b.png +2023-07-12 11:12:51,566 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmb5321e-media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 11:12:53,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:59,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:59,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:00,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:00,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:00,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:02,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:02,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:03,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:03,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:03,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:05,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:10,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:13:10,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:10,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:10,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:10,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:10,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:13,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:13,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:15,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:18,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:18,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:18,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:20,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:20,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:20,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:20,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:20,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:24,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:24,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:26,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:31,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:31,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:31,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:31,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:32,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:33,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:33,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:33,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:35,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:35,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:37,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:40,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:13:41,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:41,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:41,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:41,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:42,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:46,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:46,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:48,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:48,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:48,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:48,821 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:52,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:52,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:52,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:52,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:53,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:57,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:57,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:59,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:02,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:14:02,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:14:02,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:14:02,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:14:04,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:04,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:04,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:04,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:08,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:14:08,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:14:09,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:10,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:14:12,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:14:12,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:14:12,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:14:12,687 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:14:14,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:17,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:14:17,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:14:17,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:14:17,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:14:19,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:19,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:19,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:19,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:14:19,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:14:20,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:25,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:30,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:34,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:34,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:34,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:35,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:40,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:14:41,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:46,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:49,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:49,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:49,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:51,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:56,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:01,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:04,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:15:04,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:04,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:06,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:10,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:15:11,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_3ca3ef00974e8561788d.png with policy now +2023-07-12 11:15:11,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:11,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_7cb40a3f835111c25bc8.png with policy now +2023-07-12 11:15:11,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_1128a7fc997078f6857d.png with policy now +2023-07-12 11:15:11,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_5968f013df449c836103.png with policy now +2023-07-12 11:15:11,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:11,731 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:11,731 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:11,733 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:11,989 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ctwth8si-media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 11:15:12,015 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xpivlpww-media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 11:15:12,039 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3bqdhnsa-media/images/validation_2680_5968f013df449c836103.png +2023-07-12 11:15:12,040 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rrb9xotb-media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 11:15:14,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:14,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 11:15:14,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_5968f013df449c836103.png +2023-07-12 11:15:14,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 11:15:14,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 11:15:14,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:15:15,191 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k144eapa-media/images/validation_2680_5968f013df449c836103.png +2023-07-12 11:15:15,192 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2uo5zdfv-media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 11:15:15,197 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/toik5y3j-media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 11:15:15,208 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2fz9uxol-media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 11:15:17,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:19,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:15:19,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:19,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:22,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:22,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:25,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:27,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:31,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:31,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:31,821 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:31,823 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:32,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:34,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:34,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:15:34,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:36,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:36,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:38,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:40,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:15:41,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:41,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:41,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:41,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:43,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:47,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:47,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:48,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:49,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:49,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:51,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:51,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:51,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:51,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:54,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:58,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:58,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:59,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:01,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:01,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:01,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:01,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:04,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:04,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:04,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:09,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:09,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:10,045 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:16:10,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:11,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:11,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:11,899 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:11,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:15,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:19,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:19,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:16:19,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:20,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:21,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:21,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:22,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:22,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:22,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:22,107 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:26,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:31,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:32,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:32,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:32,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:32,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:32,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:32,615 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:34,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:16:34,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:34,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:37,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:40,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:16:42,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:42,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:42,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:42,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:42,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:43,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:43,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:47,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:47,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:47,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:47,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:47,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:16:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:49,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:53,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:54,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:54,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:58,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:03,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:04,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:04,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:04,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:08,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:10,047 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:17:14,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:19,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:19,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:19,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:19,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:24,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:29,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:34,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:34,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:34,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:35,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:40,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:17:41,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:41,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_f6facb6c2b52824b8087.png with policy now +2023-07-12 11:17:41,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_ea076148ce2508581f4a.png with policy now +2023-07-12 11:17:41,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_6b92b33ac5d9538cd14f.png with policy now +2023-07-12 11:17:41,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,838 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_09839606b108aef4ba45.png with policy now +2023-07-12 11:17:41,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:17:41,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:17:41,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:17:41,880 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:17:42,147 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/311w19x7-media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 11:17:42,157 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h6vdgr2h-media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 11:17:42,173 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jiomayv6-media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 11:17:42,211 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/45r42ksx-media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 11:17:46,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:49,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:17:49,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:17:49,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 11:17:49,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 11:17:49,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 11:17:49,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 11:17:49,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:17:49,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:49,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:49,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:50,055 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/doy6em4i-media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 11:17:50,059 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1fzuu2ea-media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 11:17:50,069 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rilbp60s-media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 11:17:50,074 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5337gwsy-media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 11:17:52,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:17:52,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:57,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:00,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:01,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:01,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:01,201 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:01,203 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:03,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:05,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:05,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:05,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:08,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:10,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:18:10,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:10,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:10,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:10,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:11,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:11,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:14,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:19,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:20,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:20,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:20,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:20,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:20,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:22,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:22,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:24,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:30,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:31,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:31,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:31,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:31,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:33,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:33,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:35,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:35,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:35,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:35,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:40,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:18:41,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:42,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:42,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:42,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:42,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:44,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:44,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:46,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:50,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:50,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:50,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:52,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:52,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:52,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:52,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:52,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:55,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:55,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:57,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:02,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:19:02,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:02,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:19:02,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:19:02,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:19:05,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:05,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:05,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:06,872 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:19:06,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:19:08,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:10,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:19:13,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:19:13,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:13,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:19:13,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:19:13,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:19:17,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:19:17,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:19:17,835 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:19:17,837 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:19:17,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:19:17,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:19:18,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:20,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:20,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:20,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:23,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:28,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:28,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:19:33,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:35,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:35,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:35,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:39,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:40,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:19:45,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:50,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:50,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:50,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:50,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:55,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:00,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:05,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:05,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:05,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:05,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:10,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:20:11,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:12,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_2a9c6e3e9705c484c43d.png with policy now +2023-07-12 11:20:12,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_951a6312bd4ff445e3bf.png with policy now +2023-07-12 11:20:12,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_cd74dc827e683d49ff49.png with policy now +2023-07-12 11:20:12,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_9d44392c5a9c6cbb63a6.png with policy now +2023-07-12 11:20:12,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:12,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:12,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:12,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:12,725 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ijw2ulkw-media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 11:20:12,727 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a8p6o5sk-media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 11:20:12,748 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o21w7tqt-media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 11:20:12,748 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mz84i902-media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 11:20:13,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:13,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 11:20:13,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 11:20:13,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 11:20:13,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 11:20:13,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:20:13,572 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u6tus951-media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 11:20:13,576 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ospjqimt-media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 11:20:13,579 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5tjc368x-media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 11:20:13,602 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3xm4k94c-media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 11:20:16,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:20,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:20,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:20,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:21,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:22,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:20:27,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:31,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:31,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:31,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:31,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:32,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:35,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:35,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:20:35,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:35,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:35,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:38,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:40,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:20:41,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:41,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:41,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:41,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:43,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:46,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:46,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:20:49,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:51,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:51,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:51,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:52,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:52,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:52,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:52,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:54,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:57,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:57,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:00,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:02,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:02,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:02,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:02,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:05,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:06,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:06,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:06,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:08,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:08,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:10,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:21:11,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:12,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:12,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:12,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:12,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:16,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:19,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:19,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:21,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:21,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:21,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:21,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:22,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:22,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:22,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:22,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:26,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:30,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:30,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:32,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:32,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:32,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:32,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:32,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:37,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:37,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:37,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:37,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:40,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:21:41,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:41,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:43,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:43,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:43,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:43,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:43,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:48,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:48,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:48,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:48,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:49,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:52,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:52,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:52,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:52,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:52,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:54,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:59,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:04,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:07,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:07,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:07,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:09,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:10,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:22:15,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:20,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:22,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:22,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:22,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:25,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:30,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:35,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:37,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:37,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:37,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:40,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:22:41,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:42,652 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_de3f2b25e35e4537c17e.png with policy now +2023-07-12 11:22:42,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_5ca25be3dc62ed26aa0d.png with policy now +2023-07-12 11:22:42,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_1328ec76d8379d97a378.png with policy now +2023-07-12 11:22:42,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_6e2b6aa87197c3eba430.png with policy now +2023-07-12 11:22:42,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:22:42,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:22:42,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:22:42,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:22:43,002 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cths0upt-media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 11:22:43,011 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gjw1rpsi-media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 11:22:43,019 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xjc0hddw-media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 11:22:43,026 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g6hy87ld-media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 11:22:46,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:47,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:22:47,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:22:47,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 11:22:47,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 11:22:47,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 11:22:47,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 11:22:47,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:22:48,341 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9y553guv-media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 11:22:48,356 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mwvvk1wp-media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 11:22:48,366 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qkk3mhyr-media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 11:22:48,367 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nwpbxe56-media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 11:22:52,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:52,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:52,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:52,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:53,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:22:57,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:58,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:02,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:02,255 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:02,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:02,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:03,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:07,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:07,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:08,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:09,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:09,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:10,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:23:12,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:12,597 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:12,597 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:12,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:14,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:19,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:21,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:21,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:22,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:22,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:23:22,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:23,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:23,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:23,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:23,004 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:25,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:30,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:32,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:32,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:33,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:33,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:33,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:33,534 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:35,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:37,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:37,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:40,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:23:40,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:43,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:43,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:43,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:43,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:43,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:43,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:46,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:51,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:52,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:52,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:54,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:54,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:54,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:54,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:54,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:54,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:56,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:01,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:04,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:24:04,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:24:04,006 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:24:04,008 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:24:05,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:24:05,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:24:07,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:08,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:08,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:08,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:10,063 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:24:12,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:13,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:24:13,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:24:13,799 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:24:13,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:24:16,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:24:16,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:24:18,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:18,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:24:18,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:24:18,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:24:18,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:24:23,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:23,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:23,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:23,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:27,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:24:27,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:24:28,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:33,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:38,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:38,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:38,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:38,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:40,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:24:44,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:49,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:53,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:53,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:53,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:54,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:59,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:04,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:08,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:08,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:08,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:09,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:10,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:25:12,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_fb1e12ca3be46bca7950.png with policy now +2023-07-12 11:25:12,875 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_cab768f5b1c4da57e828.png with policy now +2023-07-12 11:25:12,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,885 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_9ebc2d1c275cb06de97e.png with policy now +2023-07-12 11:25:12,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,898 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_62fa4b4be6b75191315d.png with policy now +2023-07-12 11:25:12,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:12,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:12,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:12,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:13,199 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gxwf057q-media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 11:25:13,216 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9zqntcs0-media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 11:25:13,229 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3z5q8hwa-media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 11:25:13,287 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dfz4offs-media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 11:25:14,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:20,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:22,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:22,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:22,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:22,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 11:25:22,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 11:25:22,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 11:25:22,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 11:25:22,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:25:23,097 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8i1l33xv-media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 11:25:23,099 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2rmua22y-media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 11:25:23,104 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ff1qg20-media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 11:25:23,107 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/00z30fnf-media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 11:25:24,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:24,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:24,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:25,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:30,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:32,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:32,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:32,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:32,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:33,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:33,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:36,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:39,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:39,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:39,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:40,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:25:42,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:42,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:42,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:42,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:42,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:44,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:44,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:47,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:52,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:52,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:52,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:52,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:52,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:54,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:57,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:02,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:03,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:03,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:03,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:03,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:06,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:06,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:08,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:10,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:26:10,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:10,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:10,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:26:13,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:13,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:13,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:13,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:13,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:17,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:17,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:18,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:24,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:24,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:24,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:24,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:24,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:26,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:26,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:26,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:26:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:29,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:34,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:34,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:34,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:34,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:34,893 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:40,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:40,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:40,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:26:40,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:41,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:41,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:41,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:26:44,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:44,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:44,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:44,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:45,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:49,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:49,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:49,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:49,631 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:51,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:51,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:56,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:57,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:57,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:57,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:02,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:27:02,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:07,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:10,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:27:12,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:12,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:12,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:13,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:18,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:23,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:27,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:27,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:27,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:28,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:33,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:38,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:40,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:27:42,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:42,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:42,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:43,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:44,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_468e29d1792ddfc31f96.png with policy now +2023-07-12 11:27:44,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_ec678e94be72a14ad671.png with policy now +2023-07-12 11:27:44,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_7a6b051b3cd7a36b4c2e.png with policy now +2023-07-12 11:27:44,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,315 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_a1852227cbd29c30acc0.png with policy now +2023-07-12 11:27:44,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:27:44,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:27:44,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:27:44,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:27:44,640 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/da6qzix1-media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 11:27:44,641 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k16v5f2r-media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 11:27:44,646 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1l1rv8a5-media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 11:27:44,664 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/40jxxe9e-media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 11:27:46,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:27:46,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 11:27:46,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 11:27:46,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 11:27:46,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 11:27:46,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:27:46,834 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zotxchec-media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 11:27:46,835 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r4oj4kbu-media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 11:27:46,843 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w96eh0h2-media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 11:27:46,851 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1qayuh72-media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 11:27:48,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:54,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:54,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:27:57,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:57,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:57,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:57,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:27:59,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:03,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:03,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:03,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:03,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:05,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:08,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:08,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:10,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:28:10,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:12,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:12,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:12,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:12,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:12,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:12,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:12,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:16,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:19,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:19,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:21,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:23,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:23,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:23,136 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:23,138 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:26,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:27,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:27,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:27,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:30,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:30,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:31,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:33,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:33,394 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:33,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:33,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:37,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:40,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:28:41,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:41,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:42,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:42,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:42,669 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:43,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:43,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:43,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:43,236 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:43,238 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:48,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:52,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:52,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:53,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:53,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:53,321 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:53,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:54,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:57,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:57,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:57,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:59,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:03,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:29:03,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:29:03,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:29:03,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:29:03,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:29:03,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:29:04,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:10,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:29:10,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:13,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:13,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:13,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:13,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:29:13,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:29:13,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:29:13,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:29:14,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:29:14,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:29:15,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:19,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:29:19,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:29:19,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:29:19,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:29:21,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:25,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:29:25,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:29:26,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:28,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:28,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:28,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:32,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:37,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:40,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:29:43,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:43,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:43,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:43,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:48,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:53,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:58,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:58,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:58,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:59,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:04,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:09,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:10,075 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:30:13,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,347 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_a1a975ef1b62e7916638.png with policy now +2023-07-12 11:30:13,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_3a9de177b9a56588f2fd.png with policy now +2023-07-12 11:30:13,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_88328b2a91d7dcdc342e.png with policy now +2023-07-12 11:30:13,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_1eadc0d7d441ac97c181.png with policy now +2023-07-12 11:30:13,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:13,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:13,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:13,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:13,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:13,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:13,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:30:13,799 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vkxysg2j-media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 11:30:13,800 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9i8vxxr-media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 11:30:13,996 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/abtycagn-media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 11:30:14,020 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/86yf18je-media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 11:30:14,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:19,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:21,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:30:21,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:21,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 11:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 11:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 11:30:21,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 11:30:21,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:30:21,518 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rhcz1xl4-media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 11:30:21,521 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/djhrhvbd-media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 11:30:21,558 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2kfg05x8-media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 11:30:21,570 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/86b8jalj-media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 11:30:23,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:25,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:28,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:28,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:28,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:30:31,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:32,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:32,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:32,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:32,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:32,491 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:36,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:40,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:30:42,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:42,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:42,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:42,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:42,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:44,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:44,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:44,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:30:44,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:30:44,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:47,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:52,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:52,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:52,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:52,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:53,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:56,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:30:56,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:58,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:59,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:59,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:59,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:02,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:02,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:02,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:02,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:03,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:08,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:08,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:09,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:10,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:31:12,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:12,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:12,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:12,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:14,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:31:14,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:31:14,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:14,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:19,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:20,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:20,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:22,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:22,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:22,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:22,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:25,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:30,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:31:30,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:31:30,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:30,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:32,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:32,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:33,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:33,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:33,279 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:33,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:35,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:40,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:31:41,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:43,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:43,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:43,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:43,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:44,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:44,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:45,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:31:45,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:31:45,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:47,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:48,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:48,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:48,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:48,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:53,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:56,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:56,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:58,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:00,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:00,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:00,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:03,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:08,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:10,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:32:14,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:15,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:15,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:15,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:19,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:24,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:29,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:30,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:35,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:40,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:32:41,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:42,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:42,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_809fb9a00d33011ef1b7.png with policy now +2023-07-12 11:32:42,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:42,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_35d816078fba6fbc5f20.png with policy now +2023-07-12 11:32:42,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:42,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_e2650e26371a6e674cfa.png with policy now +2023-07-12 11:32:43,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:43,006 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_44058a9b1a42011478fa.png with policy now +2023-07-12 11:32:43,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:32:43,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:32:43,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:32:43,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:32:43,462 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/id5cdbl9-media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 11:32:43,524 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bdgkr8c9-media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 11:32:43,539 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2wkn3hli-media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 11:32:43,605 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wexi19l1-media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 11:32:44,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:32:44,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 11:32:44,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 11:32:44,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 11:32:44,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 11:32:44,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:32:45,324 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ky8cuuqs-media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 11:32:45,351 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j0uz754x-media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 11:32:45,398 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r4xrccpl-media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 11:32:45,404 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lrzj9u28-media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 11:32:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:45,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:46,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:51,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:53,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:32:56,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:32:57,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:00,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:00,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:00,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:02,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:02,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:02,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:02,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:02,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:08,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:08,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:08,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:10,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:33:12,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:12,937 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:12,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:12,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:13,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:15,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:15,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:15,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:19,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:20,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:20,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:23,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:23,503 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:23,503 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:23,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:25,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:30,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:30,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:30,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:32,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:32,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:33,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:33,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:33,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:33,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:35,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:40,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:33:41,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:43,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:43,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:43,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:43,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:45,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:45,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:45,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:45,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:46,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:51,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:54,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:54,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:54,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:54,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:56,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:57,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:57,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:01,438 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:01,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:04,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:34:04,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:34:04,516 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:34:04,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:34:07,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:09,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:34:09,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:10,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:34:13,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:14,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:34:14,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:34:14,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:34:14,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:34:16,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:16,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:16,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:18,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:19,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:34:19,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:34:19,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:34:19,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:34:21,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:34:21,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:24,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:29,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:31,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:31,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:31,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:33,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:35,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:40,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:40,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:34:45,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:46,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:46,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:46,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:50,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:55,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:00,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:01,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:01,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:01,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:06,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:10,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:35:11,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:14,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_70b32648987a29c582d5.png with policy now +2023-07-12 11:35:14,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_aab6a0c7b6bb6712ffaa.png with policy now +2023-07-12 11:35:14,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,167 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png with policy now +2023-07-12 11:35:14,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,177 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_0eccab278b3c49ff65d0.png with policy now +2023-07-12 11:35:14,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:14,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:14,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:14,219 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:14,549 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nqm6rm4c-media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 11:35:14,552 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmrfgjp9-media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 11:35:14,571 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0vuypjzu-media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 11:35:14,574 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/64wgf1lp-media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 11:35:16,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:16,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:16,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:16,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:21,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:21,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:21,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:21,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 11:35:21,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 11:35:21,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 11:35:21,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 11:35:21,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:35:21,924 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kibcfo9b-media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 11:35:21,931 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/00twsuqf-media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 11:35:21,957 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9azkhohz-media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 11:35:21,958 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jaza5ul4-media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 11:35:23,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:26,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:31,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:32,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:32,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:32,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:33,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:33,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:33,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:33,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:33,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:33,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:37,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:40,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:35:42,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:42,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:42,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:42,999 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:43,001 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:45,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:45,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:47,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:47,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:47,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:48,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:53,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:53,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:53,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:53,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:54,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:57,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:57,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:59,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:02,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:36:02,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:02,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:03,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:03,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:03,499 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:05,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:09,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:09,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:10,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:36:11,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:13,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:13,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:13,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:13,634 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:16,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:18,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:36:18,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:18,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:21,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:21,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:21,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:24,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:24,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:24,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:24,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:27,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:32,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:33,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:36:33,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:33,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:34,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:34,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:34,321 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:34,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:37,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:40,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:36:42,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:44,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:44,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:44,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:44,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:45,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:45,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:47,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:48,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:48,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:49,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:49,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:49,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:49,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:53,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:57,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:57,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:58,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:03,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:03,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:03,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:03,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:09,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:10,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:37:14,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:19,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:19,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:19,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:20,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:25,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:30,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:34,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:34,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:34,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:40,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:37:41,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:43,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_a6f733cd68a255bc3547.png with policy now +2023-07-12 11:37:43,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_6c225fe157884e440759.png with policy now +2023-07-12 11:37:43,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,407 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_b06c0ff3bb6bd189e084.png with policy now +2023-07-12 11:37:43,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_b32e150d333150b6d60d.png with policy now +2023-07-12 11:37:43,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:37:43,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:37:43,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:37:43,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:37:43,826 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0xxgvpi5-media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 11:37:43,837 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gt2ct0cg-media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 11:37:43,839 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mkjpf5ve-media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 11:37:43,845 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9qhcrga2-media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 11:37:46,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:46,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:37:46,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 11:37:46,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 11:37:46,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 11:37:46,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 11:37:46,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:37:46,737 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7f5kywss-media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 11:37:46,737 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9du9k8n-media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 11:37:46,757 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pcf2bkya-media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 11:37:46,768 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhomi6ll-media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 11:37:49,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:49,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:49,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:52,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:53,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:37:57,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:58,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:02,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:02,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:02,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:02,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:03,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:04,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:04,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:04,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:08,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:10,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:38:10,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:10,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:12,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:12,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:12,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:12,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:13,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:18,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:19,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:19,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:19,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:22,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:22,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:22,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:22,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:22,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:22,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:24,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:30,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:33,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:33,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:33,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:33,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:34,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:34,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:35,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:35,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:35,352 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:35,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:40,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:38:40,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:43,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:43,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:43,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:43,463 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:46,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:46,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:46,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:50,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:50,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:50,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:52,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:54,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:54,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:54,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:54,071 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:57,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:58,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:58,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:39:02,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:04,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:39:04,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:39:04,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:39:04,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:39:06,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:06,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:06,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:08,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:10,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:39:10,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:39:10,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:39:14,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:14,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:39:14,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:39:14,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:39:14,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:39:19,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:19,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:39:19,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:39:19,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:39:19,625 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:39:21,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:21,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:21,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:22,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:39:22,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:39:25,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:30,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:35,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:36,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:36,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:36,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:40,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:39:41,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:46,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:51,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:51,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:51,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:51,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:56,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:01,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:06,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:06,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:06,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:06,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:10,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:40:12,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:14,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_37df37da56d5d142423a.png with policy now +2023-07-12 11:40:14,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_0ebabc283025333f43a1.png with policy now +2023-07-12 11:40:14,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,347 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_5cdc5d44821692ab8816.png with policy now +2023-07-12 11:40:14,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_37cb82e9b9e53c9e5c5d.png with policy now +2023-07-12 11:40:14,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:14,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:14,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:14,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:14,714 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yw4ld93z-media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 11:40:14,720 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ra7y4264-media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 11:40:14,731 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x0gs04uv-media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 11:40:14,792 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yvac02p2-media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 11:40:17,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:21,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:21,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:21,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:22,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:23,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:23,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:40:23,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 11:40:23,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 11:40:23,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 11:40:23,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 11:40:23,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:40:23,453 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ahdvoobk-media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 11:40:23,456 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o8el157e-media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 11:40:23,475 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tgdtcqou-media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 11:40:23,565 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqfz6szf-media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 11:40:24,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:28,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:33,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:34,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:34,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:34,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:34,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:35,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:35,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:40:36,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:36,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:36,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:39,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:40,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:40:44,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:44,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:44,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:44,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:44,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:47,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:47,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:40:49,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:51,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:51,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:51,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:54,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:54,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:54,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:54,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:54,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:59,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:59,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:00,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:04,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:04,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:04,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:04,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:05,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:06,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:06,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:06,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:10,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:41:11,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:11,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:11,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:14,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:14,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:14,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:14,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:16,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:22,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:22,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:22,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:22,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:23,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:23,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:25,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:25,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:25,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:25,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:27,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:32,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:34,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:34,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:34,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:34,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:35,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:35,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:37,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:37,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:37,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:37,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:40,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:41:42,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:45,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:45,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:45,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:45,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:47,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:47,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:48,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:50,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:50,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:50,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:50,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:52,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:52,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:52,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:53,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:58,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:59,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:59,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:42:03,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:07,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:07,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:07,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:08,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:10,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:42:14,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:19,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:22,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:22,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:22,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:24,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:29,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:34,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:37,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:37,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:37,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:39,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:40,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:42:44,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_5a82d28324cd9b502db5.png with policy now +2023-07-12 11:42:44,308 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_1df5c10fc9ddab2abd27.png with policy now +2023-07-12 11:42:44,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_a3c09a1cdfb2790cb544.png with policy now +2023-07-12 11:42:44,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_b0ea698af085bca141c6.png with policy now +2023-07-12 11:42:44,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:42:44,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:42:44,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:42:44,379 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:42:44,630 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0nfjq7b3-media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 11:42:44,632 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/39bbt4ol-media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 11:42:44,660 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nat74ho9-media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 11:42:44,677 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oo900y59-media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 11:42:45,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:47,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:42:47,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 11:42:47,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 11:42:47,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 11:42:47,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 11:42:47,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:42:48,169 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/47c50e8j-media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 11:42:48,185 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o2a6nqej-media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 11:42:48,205 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grjckrpy-media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 11:42:48,209 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ut37vrgs-media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 11:42:50,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:52,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:52,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:52,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:54,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:42:56,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:59,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:02,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:03,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:03,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:03,799 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:03,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:07,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:07,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:07,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:08,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:10,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:43:11,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:11,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:13,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:13,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:13,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:13,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:14,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:20,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:22,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:22,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:22,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:23,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:23,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:24,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:24,308 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:24,308 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:24,310 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:25,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:30,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:34,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:34,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:34,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:34,228 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:35,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:35,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:36,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:37,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:37,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:37,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:40,106 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:43:41,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:44,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:44,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:44,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:44,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:47,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:48,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:48,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:52,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:52,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:52,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:52,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:54,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:54,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:54,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:54,640 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:58,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:00,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:44:00,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:44:04,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:05,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:44:05,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:44:05,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:44:05,154 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:44:07,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:07,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:07,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:10,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:10,107 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:44:12,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:44:12,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:44:15,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:44:15,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:15,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:44:15,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:44:15,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:44:19,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:44:19,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:44:19,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:44:19,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:44:21,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:22,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:22,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:22,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:24,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:44:24,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:44:26,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:31,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:36,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:37,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:37,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:37,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:40,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:44:42,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:47,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:52,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:53,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:53,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:53,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:57,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:02,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:07,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:08,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:08,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:08,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:10,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:45:13,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:13,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,884 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_0a4eb1c4607d5a5fd63e.png with policy now +2023-07-12 11:45:13,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_a4a930c3aca9ed6b1e66.png with policy now +2023-07-12 11:45:13,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_1e810046ab391ec21457.png with policy now +2023-07-12 11:45:13,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,917 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_55e6ce392e83cd6c9775.png with policy now +2023-07-12 11:45:13,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:13,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:13,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:13,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:14,265 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0jclxal7-media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 11:45:14,272 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/la7oxeap-media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 11:45:14,274 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6copradp-media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 11:45:14,276 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7q8glz2j-media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 11:45:18,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:23,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:23,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:23,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:23,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:23,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:24,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:45:24,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:45:24,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 11:45:24,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 11:45:24,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 11:45:24,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 11:45:24,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:45:24,953 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aeud6s0x-media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 11:45:24,958 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cou4mf2w-media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 11:45:24,963 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xc50d5pf-media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 11:45:24,969 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yw17wqiy-media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 11:45:29,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:32,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:32,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:32,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:32,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:34,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:36,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:45:36,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:45:38,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:38,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:38,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:40,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:45:40,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:42,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:42,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:42,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:42,736 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:45,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:48,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:45:48,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:45:51,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:52,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:52,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:52,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:52,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:53,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:53,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:53,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:56,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:00,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:00,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:02,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:02,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:02,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:02,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:02,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:07,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:08,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:08,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:08,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:10,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:46:12,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:12,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:13,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:13,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:13,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:13,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:19,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:23,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:23,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:23,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:24,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:24,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:24,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:24,299 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:24,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:24,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:24,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:29,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:34,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:34,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:34,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:34,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:35,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:36,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:36,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:39,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:39,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:39,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:40,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:46:41,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:46,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:46,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:46,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:46,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:47,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:48,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:48,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:50,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:50,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:50,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:50,814 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:52,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:54,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:54,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:54,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:57,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:00,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:47:00,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:47:02,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:07,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:09,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:09,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:10,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:47:13,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:18,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:23,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:24,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:47:24,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:24,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:28,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:33,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:38,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:39,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:39,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:47:39,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:40,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:47:44,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:45,063 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_79746b73a2d51ab398aa.png with policy now +2023-07-12 11:47:45,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,072 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_8fd491043a269feded86.png with policy now +2023-07-12 11:47:45,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_8e4329d82c9013ca58f7.png with policy now +2023-07-12 11:47:45,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_1f22abcfa3a067c7928c.png with policy now +2023-07-12 11:47:45,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:47:45,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:47:45,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:47:45,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:47:45,512 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nc7vq0w3-media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 11:47:45,539 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ch8yrr8q-media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 11:47:45,546 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f47pc7cq-media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 11:47:45,556 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ywn97eog-media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 11:47:49,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:47:49,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 11:47:49,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 11:47:49,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 11:47:49,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 11:47:49,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:47:49,722 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eku1kuri-media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 11:47:49,729 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ta44jei-media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 11:47:49,768 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9t8jlhw7-media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 11:47:49,810 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ltk10z3i-media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 11:47:49,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:54,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:54,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:47:54,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:55,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:47:55,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:00,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:01,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:04,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:04,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:04,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:04,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:06,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:09,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:09,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:10,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:48:11,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:13,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:13,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:14,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:14,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:14,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:14,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:16,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:21,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:24,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:24,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:24,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:24,638 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:24,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:24,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:25,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:25,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:27,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:32,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:34,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:34,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:34,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:34,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:37,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:37,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:38,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:39,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:39,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:40,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:48:43,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:45,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:45,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:45,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:45,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:49,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:49,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:49,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:54,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:48:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:54,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:55,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:55,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:55,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:55,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:00,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:01,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:49:01,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:49:05,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:06,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:49:06,308 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:49:06,308 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:49:06,310 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:10,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:49:10,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:10,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:10,732 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:10,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:13,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:49:13,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:49:15,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:49:15,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:15,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:49:15,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:49:15,978 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:20,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:49:20,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:49:20,671 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:49:20,673 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:21,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:25,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:49:25,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:49:25,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:25,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:25,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:26,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:31,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:36,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:40,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:49:40,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:40,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:40,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:41,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:46,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:51,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:55,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:55,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:55,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:56,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:01,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:06,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:10,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:50:10,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:10,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:10,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:12,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:15,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_0393c6fc4db3e318d3d3.png with policy now +2023-07-12 11:50:15,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_4fee48092f2ede4f3c25.png with policy now +2023-07-12 11:50:15,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_ba69b7651ab0dcd73bdc.png with policy now +2023-07-12 11:50:15,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_af052cd0179c44de7cb7.png with policy now +2023-07-12 11:50:15,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:15,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:15,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:15,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:15,341 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ccf3zvw8-media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 11:50:15,343 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/07h7keny-media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 11:50:15,363 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qg7wchih-media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 11:50:15,378 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ifg0kdz2-media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 11:50:17,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:23,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:24,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:25,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:25,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:25,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:25,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:50:26,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:50:26,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 11:50:26,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 11:50:26,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 11:50:26,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 11:50:26,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:50:26,343 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ulagkxk-media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 11:50:26,344 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m8fxd9dx-media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 11:50:26,346 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3xz7y70-media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 11:50:26,375 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z0e008uj-media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 11:50:29,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:34,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:34,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:34,779 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:34,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:34,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:50:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:50:40,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:50:40,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:40,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:40,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:40,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:45,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:45,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:45,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:45,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:45,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:50,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:50:50,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:50:51,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:56,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:56,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:56,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:56,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:56,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:56,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:56,072 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:57,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:02,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:02,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:02,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:06,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:06,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:06,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:06,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:08,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:10,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:51:11,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:11,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:11,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:51:13,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:14,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:14,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:16,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:16,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:16,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:16,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:18,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:24,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:26,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:26,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:26,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:26,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:26,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:51:27,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:27,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:27,082 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:27,084 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:29,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:34,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:37,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:37,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:37,428 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:37,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:38,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:38,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:40,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:51:40,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:42,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:42,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:42,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:51:45,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:47,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:47,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:47,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:47,656 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:50,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:50,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:51,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:52,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:52,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:52,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:52,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:57,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:57,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:57,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:57,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:02,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:02,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:52:02,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:52:07,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:10,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:52:12,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:12,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:12,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:12,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:17,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:22,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:27,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:27,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:27,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:28,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:33,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:38,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:40,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:52:42,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:42,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:42,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:43,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:46,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_a5847a1c9ac576d3c242.png with policy now +2023-07-12 11:52:46,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_d0edf809dce9067d913a.png with policy now +2023-07-12 11:52:46,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_1176c8bf7fff463bf9c6.png with policy now +2023-07-12 11:52:46,749 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_f3f42317c0356c47ff20.png with policy now +2023-07-12 11:52:46,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:52:46,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:52:46,789 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:52:46,791 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:52:47,041 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xd4oc6qt-media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 11:52:47,059 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1h3w6sog-media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 11:52:47,076 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i87ajoq2-media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 11:52:47,204 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xc91uqac-media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 11:52:48,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:50,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:52:50,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 11:52:50,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 11:52:50,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 11:52:50,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 11:52:50,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:52:51,035 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/74nu2nq5-media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 11:52:51,040 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1im8kr8-media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 11:52:51,041 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/44as5a81-media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 11:52:51,059 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3trgye8s-media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 11:52:53,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:56,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:52:57,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:57,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:57,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:59,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:02,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:04,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:05,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:05,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:05,820 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:05,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:09,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:10,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:53:12,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:53:12,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:12,943 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:53:14,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:14,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:15,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:15,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:15,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:15,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:15,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:20,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:25,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:25,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:25,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:25,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:26,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:26,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:26,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:27,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:27,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:53:31,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:35,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:35,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:35,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:35,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:36,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:38,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:38,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:40,127 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:53:42,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:43,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:53:43,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:43,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:53:46,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:46,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:46,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:46,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:47,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:51,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:51,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:53,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:57,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:57,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:57,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:57,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:59,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:59,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:53:59,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:59,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:03,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:54:03,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:54:04,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:07,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:54:07,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:54:07,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:54:07,173 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:54:09,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:10,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:54:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:54:14,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:14,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:15,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:54:15,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:54:17,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:54:17,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:54:17,982 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:54:17,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:54:19,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:22,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:54:22,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:54:22,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:54:22,597 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:54:25,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:27,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:54:27,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:54:29,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:29,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:54:29,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:30,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:35,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:40,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:54:41,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:44,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:44,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:46,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:51,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:56,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:59,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:54:59,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:59,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:01,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:06,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:10,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:55:12,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:14,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:14,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:14,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:16,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_49d3966f62f54fe9ca62.png with policy now +2023-07-12 11:55:16,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_a708be3afa6ae1a86e60.png with policy now +2023-07-12 11:55:16,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_7e7433279443a3a2bfa6.png with policy now +2023-07-12 11:55:16,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_2d3f82af7844062f4b35.png with policy now +2023-07-12 11:55:16,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:16,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:16,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:16,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:16,980 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u790hx6o-media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 11:55:16,998 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e3qvq5ik-media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 11:55:17,021 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1ymaqvb1-media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 11:55:17,083 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xxqd9my6-media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 11:55:17,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:23,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:26,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:27,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:55:27,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:55:27,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 11:55:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 11:55:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 11:55:27,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 11:55:27,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:55:27,791 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9q8rsbr-media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 11:55:27,814 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s2pxx9du-media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 11:55:27,818 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gsggktdi-media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 11:55:27,821 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2tczu5pt-media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 11:55:28,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:29,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:29,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:29,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:34,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:35,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:35,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:35,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:35,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:39,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:39,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:55:39,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:55:40,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:55:44,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:44,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:44,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:44,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:46,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:46,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:46,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:46,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:50,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:51,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:55:51,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:55:55,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:56,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:56,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:56,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:56,405 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:59,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:59,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:59,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:01,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:03,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:03,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:06,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:06,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:06,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:06,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:06,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:10,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:56:12,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:14,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:14,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:14,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:15,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:15,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:16,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:16,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:16,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:16,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:17,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:22,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:26,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:26,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:26,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:26,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:27,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:27,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:28,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:29,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:29,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:29,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:33,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:36,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:36,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:36,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:36,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:38,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:39,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:39,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:40,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:56:44,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:44,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:44,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:44,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:46,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:46,772 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:46,772 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:46,774 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:49,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:51,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:51,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:51,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:51,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:51,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:51,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:55,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:59,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:59,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:59,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:00,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:03,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:57:05,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:10,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:57:11,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:14,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:14,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:14,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:16,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:21,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:26,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:29,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:29,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:29,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:31,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:36,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:40,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:57:42,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:44,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:44,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:44,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:45,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_dfe649466fc1947dac85.png with policy now +2023-07-12 11:57:45,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_2975002e2263b7b13da6.png with policy now +2023-07-12 11:57:45,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_462a59d1382cf36adaec.png with policy now +2023-07-12 11:57:45,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_9c5803c959d6e590aeae.png with policy now +2023-07-12 11:57:45,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:57:45,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:57:45,973 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:57:45,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:57:46,247 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w1b9zd0x-media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 11:57:46,259 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7zunlxji-media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 11:57:46,273 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nhoxa207-media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 11:57:46,289 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a5exp6qt-media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 11:57:47,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:52,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:57:52,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:57:52,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 11:57:52,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 11:57:52,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 11:57:52,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 11:57:52,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:57:52,588 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4m4pkq5c-media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 11:57:52,602 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/axex8uqg-media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 11:57:52,615 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/953wek8n-media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 11:57:52,655 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4vbv94n1-media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 11:57:53,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:56,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:57:59,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:59,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:59,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:59,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:04,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:04,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:05,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:05,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:05,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:05,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:10,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:58:10,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:58:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:58:15,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:58:15,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:15,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:15,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:15,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:15,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:16,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:16,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:20,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:26,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:26,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:26,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:26,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:26,786 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:28,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:28,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:30,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:58:30,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:58:30,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:58:32,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:37,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:37,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:37,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:37,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:37,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:40,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:58:40,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:40,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:43,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:45,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:58:45,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:58:45,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:58:46,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:46,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:46,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:46,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:48,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:52,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:52,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:53,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:57,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:57,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:57,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:57,108 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:59,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:00,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:00,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:00,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:04,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:04,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:59:04,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:59:07,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:59:07,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:59:07,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:59:07,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:59:09,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:10,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:59:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:15,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:15,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:15,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:16,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:59:16,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:59:18,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:59:18,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:59:18,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:59:18,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:59:20,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:22,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:59:22,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:59:22,429 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:59:22,431 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:59:26,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:28,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:59:28,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:59:30,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:30,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:30,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:31,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:36,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:40,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:59:42,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:45,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:45,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:45,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:47,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:52,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:57,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:00,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:00,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:00,207 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:02,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:07,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:10,144 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:00:13,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:15,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:15,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:15,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:16,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_c18b3c294fb48283f9a6.png with policy now +2023-07-12 12:00:16,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_e995c389d61465ef369b.png with policy now +2023-07-12 12:00:16,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_a6d367b1ce47fd112b6d.png with policy now +2023-07-12 12:00:16,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_556f808727f18f3fb7b6.png with policy now +2023-07-12 12:00:16,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:16,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:16,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:16,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:16,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:00:16,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:00:16,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:00:16,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:00:16,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:00:16,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:00:17,055 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rjv23evm-media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:00:17,070 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j4wm7qr7-media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:00:17,071 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u07zm7bo-media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:00:17,096 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sg7wbrnb-media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:00:17,450 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/14s1hk3v-media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:00:17,456 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/joxct5od-media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:00:17,795 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/12wbqli7-media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:00:17,801 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hkzx7la1-media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:00:18,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:24,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:26,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:28,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:00:30,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:31,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:31,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:31,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:35,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:36,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:36,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:36,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:36,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:40,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:00:40,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:00:40,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:00:41,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:46,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:46,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:46,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:46,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:46,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:46,120 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:46,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:47,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:52,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:53,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:00:53,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:00:55,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:55,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:55,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:55,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:58,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:01,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:01,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:01,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:03,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:05,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:05,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:06,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:06,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:06,447 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:06,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:09,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:10,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:01:15,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:16,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:16,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:16,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:17,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:17,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:17,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:17,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:17,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:17,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:20,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:25,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:27,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:27,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:27,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:27,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:29,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:29,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:31,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:32,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:32,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:32,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:37,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:38,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:38,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:38,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:38,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:40,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:01:41,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:41,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:42,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:47,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:47,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:47,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:48,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:48,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:48,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:48,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:53,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:53,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:53,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:53,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:53,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:53,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:53,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:58,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:02,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:02,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:02,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:04,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:05,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:02:09,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:10,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:02:15,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:17,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:17,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:17,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:21,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:26,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:31,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:32,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:32,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:32,236 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:36,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:40,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:02:42,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:47,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:47,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:47,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:47,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:47,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_bcac2792455fc0c610c6.png with policy now +2023-07-12 12:02:47,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,698 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_0e8bce2d57e7e047c357.png with policy now +2023-07-12 12:02:47,708 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,708 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_fe5c449ea1876a5e69ca.png with policy now +2023-07-12 12:02:47,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_1d174f0fc051b4fa542a.png with policy now +2023-07-12 12:02:47,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:02:47,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:02:47,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:02:47,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:02:48,073 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ce8epkca-media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:02:48,116 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ugy3jxxi-media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:02:48,152 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8nq1c9x7-media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:02:48,174 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d8d71ko6-media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:02:52,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:53,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:02:53,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:02:53,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:02:53,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:02:53,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:02:53,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:02:53,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:02:54,048 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4163l9ir-media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:02:54,057 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dxoas7v0-media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:02:54,094 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0sux7973-media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:02:54,331 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ubfm9zai-media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:02:57,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:57,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:02,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:02,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:02,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:02,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:05,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:07,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:07,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:07,086 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:07,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:08,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:10,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:03:13,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:16,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:16,235 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:16,236 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:16,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:17,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:17,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:17,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:17,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:17,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:19,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:24,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:26,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:26,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:26,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:26,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:29,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:29,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:30,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:32,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:32,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:32,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:36,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:37,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:37,440 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:37,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:37,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:40,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:03:41,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:41,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:42,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:47,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:47,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:47,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:47,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:47,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:47,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:47,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:47,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:52,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:54,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:54,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:57,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:59,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:59,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:59,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:59,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:02,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:02,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:02,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:03,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:06,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:04:06,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:04:08,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:09,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:04:09,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:04:09,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:04:09,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:10,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:04:14,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:17,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:17,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:17,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:18,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:04:18,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:04:19,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:04:19,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:19,313 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:04:19,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:04:19,316 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:23,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:04:23,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:04:23,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:04:23,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:25,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:30,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:04:30,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:04:30,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:32,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:32,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:32,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:35,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:40,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:04:41,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:46,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:47,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:47,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:47,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:51,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:56,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:01,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:02,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:02,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:02,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:07,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:10,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:05:13,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:17,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:17,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:17,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:18,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_9b8a7d4902a70bbe3529.png with policy now +2023-07-12 12:05:18,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,108 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_948197ed3bdc8486cb6a.png with policy now +2023-07-12 12:05:18,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_7ab6e23c2268ebda9ab0.png with policy now +2023-07-12 12:05:18,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_ad77bbc22d61066c6ce9.png with policy now +2023-07-12 12:05:18,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:18,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:18,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:18,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:18,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:05:18,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:05:18,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:05:18,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:05:18,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:05:18,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:05:18,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:05:18,533 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z857hs5f-media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:05:18,534 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fimltdk8-media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:05:18,540 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k2ckmw89-media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:05:18,551 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhlu0xn2-media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:05:18,912 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ezs2obf-media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:05:18,925 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zn1z5r4i-media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:05:18,972 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/htsbku3n-media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:05:19,319 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9b0brssr-media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:05:23,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:27,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:28,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:30,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:05:32,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:32,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:32,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:34,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:36,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:36,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:36,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:36,464 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:05:39,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:40,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:05:42,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:05:42,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:05:45,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:45,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:45,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:45,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:45,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:05:48,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:48,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:48,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:51,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:54,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:05:54,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:05:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:56,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:56,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:56,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:56,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:02,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:03,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:03,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:03,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:06,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:06,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:07,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:07,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:07,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:07,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:07,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:10,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:06:12,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:17,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:17,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:17,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:17,425 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:18,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:18,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:18,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:18,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:18,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:18,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:23,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:27,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:27,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:27,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:27,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:28,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:30,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:30,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:33,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:33,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:33,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:34,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:37,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:37,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:37,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:37,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:39,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:40,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:06:42,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:42,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:44,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:47,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:47,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:47,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:47,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:48,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:48,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:48,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:50,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:52,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:52,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:52,300 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:52,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:54,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:54,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:55,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:00,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:03,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:03,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:03,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:06,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:10,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:07:12,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:17,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:18,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:18,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:18,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:22,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:27,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:32,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:33,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:33,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:33,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:37,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:40,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:07:43,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:46,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,719 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_cb1bd764507bb8c18d8a.png with policy now +2023-07-12 12:07:46,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_8578482fbd741b1fda1e.png with policy now +2023-07-12 12:07:46,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,814 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_98e0fea80621818b19b9.png with policy now +2023-07-12 12:07:46,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_5371e29c2e2d3a7d3242.png with policy now +2023-07-12 12:07:46,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:07:46,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:07:46,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:07:46,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:07:47,159 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/txe81qoa-media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:07:47,229 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j7ihcke2-media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:07:47,265 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0wx8ct10-media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:07:47,272 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/canyduet-media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:07:48,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:48,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:48,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:48,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:54,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:55,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:07:55,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:07:55,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:07:55,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:07:55,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:07:55,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:07:55,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:07:55,660 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mzblicci-media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:07:55,670 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4t8luk82-media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:07:55,694 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iwa1v40l-media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:07:55,714 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kxp6z904-media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:07:56,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:07:59,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:03,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:03,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:03,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:04,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:06,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:06,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:06,877 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:06,879 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:07,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:07,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:10,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:08:10,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:15,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:16,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:16,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:16,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:16,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:18,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:18,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:18,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:19,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:19,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:20,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:26,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:26,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:26,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:26,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:26,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:31,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:31,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:31,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:33,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:33,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:33,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:37,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:37,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:37,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:37,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:37,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:40,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:08:42,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:43,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:43,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:47,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:47,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:47,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:47,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:48,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:49,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:49,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:49,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:54,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:55,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:55,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:57,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:57,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:57,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:57,199 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:59,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:04,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:04,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:04,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:04,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:07,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:09:07,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:09:07,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:09:07,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:09:07,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:09:07,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:09:09,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:10,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:09:15,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:18,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:09:18,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:09:18,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:09:18,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:09:19,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:19,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:19,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:19,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:09:19,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:09:20,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:23,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:09:23,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:09:23,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:09:23,192 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:09:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:31,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:09:31,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:09:31,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:34,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:34,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:34,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:37,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:40,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:09:43,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:48,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:49,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:49,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:49,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:53,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:58,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:03,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:04,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:04,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:04,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:08,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:10,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:10:14,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:16,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_3d0ad7c57f2af82f8bbc.png with policy now +2023-07-12 12:10:16,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_8f1fe1b72bfe04a0b014.png with policy now +2023-07-12 12:10:16,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,965 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_aa4b86f6ab2a980e6525.png with policy now +2023-07-12 12:10:16,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_27c9042e9f887913b3c0.png with policy now +2023-07-12 12:10:17,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:17,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:17,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:17,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:10:17,394 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8chtjdl5-media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:10:17,398 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hjwj7aji-media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:10:17,400 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fmb4lavj-media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:10:17,418 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ydejr775-media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:10:19,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:19,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:19,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:19,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:20,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:10:20,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:10:20,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:10:20,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:10:20,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:10:20,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:10:20,667 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5h6e567u-media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:10:20,669 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g7z8b3qz-media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:10:20,670 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q5jqkch9-media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:10:20,675 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5cosu0hu-media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:10:24,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:27,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:29,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:32,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:10:34,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:34,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:34,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:35,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:36,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:36,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:36,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:36,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:10:40,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:10:41,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:44,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:10:44,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:10:46,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:46,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:46,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:46,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:46,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:10:49,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:49,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:49,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:51,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:56,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:10:56,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:10:56,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:56,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:56,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:56,542 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:56,544 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:02,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:04,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:04,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:04,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:06,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:06,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:06,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:06,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:07,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:08,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:08,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:10,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:11:13,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:16,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:16,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:16,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:16,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:18,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:19,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:19,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:19,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:20,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:20,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:23,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:26,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:26,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:26,880 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:26,882 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:29,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:32,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:32,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:34,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:34,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:34,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:34,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:37,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:37,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:37,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:37,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:40,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:11:40,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:44,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:44,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:45,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:47,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:47,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:47,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:47,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:49,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:49,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:49,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:51,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:52,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:52,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:52,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:52,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:56,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:56,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:56,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:01,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:04,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:04,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:04,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:07,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:10,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:12:13,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:18,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:19,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:19,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:19,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:23,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:28,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:33,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:34,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:34,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:34,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:38,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:40,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:12:44,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:46,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_2d10951a0f8b9aed5734.png with policy now +2023-07-12 12:12:46,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_42b541158c5e9a2b90ee.png with policy now +2023-07-12 12:12:46,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_ba216e11217e75e9719b.png with policy now +2023-07-12 12:12:46,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,393 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_885722981f42772a687d.png with policy now +2023-07-12 12:12:46,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:12:46,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:12:46,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:12:46,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:12:46,676 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sijfrl8w-media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:12:46,680 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/riw9ty4b-media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:12:46,704 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w258tvlk-media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:12:46,725 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ehpj260-media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:12:49,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:49,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:49,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:49,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:54,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:56,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:12:56,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:12:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:12:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:12:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:12:56,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:12:56,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:12:56,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:12:57,277 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/46b39vmh-media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:12:57,284 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hg5oa619-media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:12:57,300 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mwtdoh5z-media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:12:57,303 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tl0qjqrx-media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:12:59,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:04,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:04,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:04,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:05,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:05,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:05,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:05,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:05,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:08,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:08,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:10,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:13:10,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:15,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:15,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:15,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:15,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:15,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:20,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:20,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:20,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:20,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:20,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:20,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:25,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:25,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:25,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:25,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:26,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:31,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:32,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:32,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:35,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:35,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:35,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:36,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:36,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:36,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:36,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:37,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:40,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:13:42,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:46,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:46,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:46,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:46,632 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:47,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:51,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:51,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:51,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:53,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:56,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:56,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:56,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:56,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:57,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:57,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:58,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:04,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:06,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:06,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:06,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:06,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:14:06,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:14:06,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:14:06,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:14:09,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:14:09,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:14:09,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:10,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:14:15,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:17,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:14:17,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:14:17,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:14:17,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:14:21,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:21,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:14:21,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:14:21,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:21,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:21,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:22,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:14:22,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:14:22,279 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:14:22,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:14:26,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:31,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:33,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:14:33,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:14:36,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:36,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:36,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:37,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:40,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:14:43,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:48,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:51,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:51,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:51,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:53,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:58,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:03,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:06,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:06,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:06,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:09,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:10,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:15:14,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:16,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_3f89f7c3f5ff7126de27.png with policy now +2023-07-12 12:15:16,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_ec96bb36cf4e66588cd4.png with policy now +2023-07-12 12:15:16,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_e0997462fe5eec3ca9d3.png with policy now +2023-07-12 12:15:16,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_0a30f7284bacae8356b3.png with policy now +2023-07-12 12:15:16,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:16,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:16,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:16,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:16,831 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zmipigev-media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:15:16,844 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bczgeohh-media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:15:16,869 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/471spo3a-media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:15:16,937 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ioot3bx-media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:15:19,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:21,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:15:21,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:21,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:15:21,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:15:21,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:15:21,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:15:21,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:15:21,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:21,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:21,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:22,145 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qbc8kinp-media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:15:22,157 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7offw2l-media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:15:22,159 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blzf4ess-media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:15:22,160 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uck3bxjx-media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:15:25,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:26,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:31,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:33,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:36,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:36,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:36,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:36,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:36,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:36,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:36,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:36,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:40,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:15:41,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:45,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:15:45,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:45,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:45,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:45,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:45,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:47,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:53,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:53,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:53,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:53,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:56,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:56,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:56,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:56,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:57,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:15:57,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:58,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:04,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:07,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:07,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:07,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:07,425 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:08,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:08,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:08,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:09,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:09,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:09,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:10,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:16:15,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:17,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:17,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:17,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:17,844 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:20,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:22,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:22,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:23,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:23,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:23,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:25,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:28,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:28,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:28,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:28,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:31,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:34,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:34,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:36,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:38,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:38,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:38,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:38,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:38,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:38,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:38,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:40,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:16:41,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:46,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:46,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:47,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:48,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:48,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:48,600 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:48,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:53,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:53,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:53,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:53,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:53,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:53,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:53,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:53,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:58,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:58,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:58,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:03,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:08,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:17:08,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:08,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:09,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:10,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:17:14,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:19,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:23,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:23,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:17:23,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:24,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:29,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:34,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:38,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:38,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:17:38,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:40,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:17:40,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:45,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:47,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_fd71982266285f250c3c.png with policy now +2023-07-12 12:17:47,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_2268465f1770440e1c31.png with policy now +2023-07-12 12:17:47,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_94f80d2b1ffdc85aabfa.png with policy now +2023-07-12 12:17:47,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_ccf35b7fa382262ad056.png with policy now +2023-07-12 12:17:47,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:17:47,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:17:47,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:17:47,821 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:17:48,086 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dljh5a3o-media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:17:48,092 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mv5p4tkr-media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:17:48,112 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a2m9e8ia-media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:17:48,118 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ho5wvbyg-media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:17:50,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:53,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:53,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:56,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:57,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:17:58,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:17:58,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:17:58,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:17:58,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:17:58,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:17:58,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:17:58,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:17:58,892 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wqt4ae0r-media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:17:58,902 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/en49xxp1-media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:17:58,910 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xp40gmfc-media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:17:58,917 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eu5dnif0-media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:18:01,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:06,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:06,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:06,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:06,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:07,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:08,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:08,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:10,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:18:10,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:10,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:12,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:16,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:16,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:16,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:16,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:17,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:22,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:22,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:23,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:24,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:18:24,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:24,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:27,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:27,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:27,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:27,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:29,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:34,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:34,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:34,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:39,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:39,075 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:39,075 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:39,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:39,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:18:39,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:39,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:40,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:18:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:45,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:46,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:46,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:49,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:49,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:49,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:49,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:50,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:54,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:54,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:18:54,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:55,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:58,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:58,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:00,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:00,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:00,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:01,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:06,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:09,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:09,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:09,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:09,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:09,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:09,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:09,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:10,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:19:10,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:19:10,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:12,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:17,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:20,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:20,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:20,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:20,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:22,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:22,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:19:22,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:24,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:24,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:24,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:24,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:24,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:24,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:24,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:28,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:33,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:34,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:19:34,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:38,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:39,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:39,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:39,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:40,187 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:19:44,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:49,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:54,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:54,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:54,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:54,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:59,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:04,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:09,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:09,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:09,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:09,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:10,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:20:15,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:18,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_11bcda97ccccef6067ba.png with policy now +2023-07-12 12:20:18,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_ba511e69b1b4757bb114.png with policy now +2023-07-12 12:20:18,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_32467ef9d42b5e1e1a33.png with policy now +2023-07-12 12:20:18,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_c49f0237624dac615f66.png with policy now +2023-07-12 12:20:18,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:18,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:18,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:18,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:19,182 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eaiwd0yw-media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:20:20,218 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kmxnke0i-media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:20:20,521 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/az3mw0ve-media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:20:20,619 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7bsjeyd-media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:20:20,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:23,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:20:23,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:20:23,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:20:23,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:20:23,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:20:23,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:20:23,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:20:23,664 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xwfdjk5j-media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:20:23,665 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5oqg297a-media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:20:23,666 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9bkxkooz-media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:20:23,667 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6iy29cl2-media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:20:24,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:24,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:24,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:26,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:28,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:31,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:35,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:20:36,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:37,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:37,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:37,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:37,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:39,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:39,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:39,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:40,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:20:42,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:47,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:47,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:47,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:47,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:47,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:20:47,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:20:48,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:53,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:55,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:55,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:55,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:58,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:58,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:58,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:58,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:59,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:20:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:04,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:08,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:08,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:08,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:08,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:09,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:10,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:21:11,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:11,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:11,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:11,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:11,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:15,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:19,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:19,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:19,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:19,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:21,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:23,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:23,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:26,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:26,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:26,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:26,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:29,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:29,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:29,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:29,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:31,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:35,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:35,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:36,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:39,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:39,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:39,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:39,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:40,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:21:41,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:41,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:41,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:42,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:47,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:49,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:49,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:49,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:49,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:53,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:54,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:54,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:54,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:54,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:56,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:56,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:56,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:58,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:59,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:59,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:22:03,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:08,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:10,192 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:22:11,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:11,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:11,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:13,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:18,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:23,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:26,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:26,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:26,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:29,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:34,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:39,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:40,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:22:41,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:41,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:41,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:44,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:48,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,209 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_5189a1941e0ee2300ecf.png with policy now +2023-07-12 12:22:48,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_dce90c9fdac2309feb19.png with policy now +2023-07-12 12:22:48,228 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,228 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_0d199df0c17d88c8726a.png with policy now +2023-07-12 12:22:48,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_626d6d1dadf7aad42361.png with policy now +2023-07-12 12:22:48,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:22:48,278 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:22:48,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:22:48,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:22:48,566 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rr17uxev-media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:22:48,582 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hckqtfvy-media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:22:48,589 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gbakifvj-media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:22:48,622 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3myhupc6-media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:22:50,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:55,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:56,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:56,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:56,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:57,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:00,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:00,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:23:00,428 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kntt3qh8-media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:23:00,438 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/css5rft9-media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:23:00,441 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/82lwht82-media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:23:00,463 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rualo752-media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:23:01,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:06,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:06,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:06,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:06,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:06,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:10,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:23:11,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:23:11,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:11,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:11,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:12,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:12,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:16,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:16,669 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:16,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:16,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:17,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:23,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:24,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:24,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:26,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:26,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:26,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:26,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:26,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:26,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:28,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:34,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:36,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:36,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:36,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:36,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:36,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:36,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:39,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:40,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:23:41,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:23:41,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:41,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:45,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:47,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:47,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:47,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:47,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:48,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:48,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:51,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:56,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:57,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:23:57,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:57,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:58,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:58,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:58,137 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:58,139 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:00,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:24:00,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:02,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:08,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:08,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:24:08,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:24:08,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:24:08,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:10,197 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:24:12,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:24:12,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:13,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:13,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:24:13,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:13,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:24:18,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:19,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:24:19,192 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:24:19,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:24:19,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:24,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:24,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:24:24,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:24:24,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:24:24,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:24,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:24:24,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:24:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:28,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:24:29,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:34,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:36,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:39,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:40,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:24:43,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:43,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:24:43,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:24:45,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:50,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:55,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:58,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:58,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:01,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:06,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:10,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:25:12,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:13,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:13,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:17,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:18,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_7b43af642a476832b780.png with policy now +2023-07-12 12:25:18,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,622 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_76db5a33fad883eea4bc.png with policy now +2023-07-12 12:25:18,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_366cee911e557f14b4e5.png with policy now +2023-07-12 12:25:18,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_95e48c47882f1ab9ab69.png with policy now +2023-07-12 12:25:18,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:18,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:18,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:18,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:18,920 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/97p627zh-media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:25:18,933 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tu10k09b-media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:25:18,935 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sxrnbcsc-media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:25:18,977 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/omnjhs6f-media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:25:22,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:24,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:25:24,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:25:24,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:25:24,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:25:24,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:25:24,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:25:24,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:25:25,188 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1c7qx1xo-media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:25:25,200 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d036ppk6-media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:25:25,233 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/flsqtx8a-media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:25:25,235 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dyx1mfgr-media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:25:28,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:28,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:25:28,566 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:33,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:36,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:25:37,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:37,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:37,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:37,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:39,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:40,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:25:43,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:25:43,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:43,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:44,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:47,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:47,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:47,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:47,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:25:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:25:50,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:55,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:58,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:58,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:58,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:58,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:58,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:25:58,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:58,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:00,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:00,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:00,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:06,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:08,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:08,900 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:08,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:08,902 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:10,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:26:11,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:12,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:12,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:13,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:26:13,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:13,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:16,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:19,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:19,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:19,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:19,391 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:21,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:25,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:25,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:27,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:29,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:26:29,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:29,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:29,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:29,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:29,690 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:29,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:32,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:37,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:37,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:37,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:40,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:26:40,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:40,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:40,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:40,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:42,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:44,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:44,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:47,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:49,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:49,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:50,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:50,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:50,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:50,653 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:52,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:55,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:55,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:55,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:55,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:57,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:59,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:26:59,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:59,219 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:01,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:27:01,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:27:03,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:08,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:10,204 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:27:14,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:14,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:14,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:14,229 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:19,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:24,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:29,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:29,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:29,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:29,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:34,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:39,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:40,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:27:44,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:44,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:44,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:44,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:49,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:49,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_c7e2d3867abfa6c74ec7.png with policy now +2023-07-12 12:27:49,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_da274c6af8f1eb4795bd.png with policy now +2023-07-12 12:27:49,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_3fff3d3522577450efef.png with policy now +2023-07-12 12:27:49,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_949364012c78ad26092f.png with policy now +2023-07-12 12:27:49,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:27:49,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:27:49,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:27:49,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:27:50,135 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1jcwp3dw-media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:27:50,137 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tvxmkdsd-media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:27:50,151 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oa0qj1wz-media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:27:50,171 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8hkk9nx5-media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:27:54,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:59,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:59,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:59,395 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:00,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:00,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:01,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:01,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:01,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:28:01,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:28:01,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:28:01,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:28:01,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:28:01,991 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ahvgb5ys-media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:28:01,993 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kh91i3vt-media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:28:02,002 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpaz0iny-media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:28:02,007 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nwup7i5f-media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:28:05,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:08,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:08,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:08,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:08,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:10,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:28:11,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:13,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:13,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:15,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:28:15,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:28:15,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:16,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:18,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:18,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:18,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:18,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:21,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:25,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:25,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:27,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:28,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:28,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:28,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:28,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:30,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:28:30,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:28:30,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:33,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:37,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:37,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:38,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:38,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:38,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:38,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:38,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:40,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:28:44,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:45,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:28:45,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:28:45,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:48,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:48,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:48,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:48,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:49,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:49,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:49,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:55,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:59,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:59,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:59,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:59,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:00,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:00,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:00,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:00,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:01,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:01,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:06,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:10,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:29:11,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:29:11,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:29:11,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:29:11,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:12,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:13,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:13,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:15,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:15,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:15,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:17,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:21,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:29:21,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:29:21,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:29:21,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:22,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:25,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:25,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:26,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:29:26,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:29:26,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:29:26,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:27,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:31,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:31,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:31,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:33,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:38,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:40,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:29:43,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:46,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:46,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:46,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:48,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:53,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:58,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:01,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:01,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:01,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:04,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:09,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:10,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:30:15,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:16,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:16,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:16,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:20,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:20,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_86a486a36860b5c6245c.png with policy now +2023-07-12 12:30:20,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,616 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_18ea565e486f4bef2bbd.png with policy now +2023-07-12 12:30:20,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,626 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_dfadefec7519a6555efa.png with policy now +2023-07-12 12:30:20,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_26cc58965abb0300912c.png with policy now +2023-07-12 12:30:20,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:20,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:30:20,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:30:20,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:30:20,928 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2eb7nl4q-media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:30:20,934 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j4sqpmdd-media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:30:20,939 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l8h8fxsf-media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:30:20,941 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ux6ylmrx-media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:30:25,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:26,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:30:26,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:30:26,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:30:26,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:30:26,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:30:26,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:30:26,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:30:26,693 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qdrsthso-media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:30:26,721 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/642cduad-media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:30:26,728 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/br5y3njn-media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:30:26,768 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wn0aw9xq-media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:30:30,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:30,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:31,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:31,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:31,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:36,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:38,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:30:39,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:39,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:30:39,799 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:30:39,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:30:40,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:30:42,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:46,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:46,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:46,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:47,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:49,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:49,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:30:49,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:30:49,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:30:50,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:30:50,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:30:53,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:58,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:00,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:00,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:00,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:00,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:02,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:02,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:02,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:03,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:09,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:10,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:31:10,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:10,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:10,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:10,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:14,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:14,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:15,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:17,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:17,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:17,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:20,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:20,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:20,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:20,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:20,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:25,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:26,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:26,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:31,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:31,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:31,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:31,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:31,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:32,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:32,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:32,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:38,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:38,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:40,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:31:41,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:41,109 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:41,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:41,111 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:42,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:47,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:48,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:48,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:48,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:51,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:51,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:51,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:51,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:53,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:56,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:56,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:56,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:56,549 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:59,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:02,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:32:02,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:32:03,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:03,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:03,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:04,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:09,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:10,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:32:15,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:18,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:18,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:18,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:20,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:25,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:30,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:33,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:33,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:33,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:35,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:40,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:32:41,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:46,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:48,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:48,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:48,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:51,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_d1566b7d6fc6fface55f.png with policy now +2023-07-12 12:32:51,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_3375e10d6215e10e1662.png with policy now +2023-07-12 12:32:51,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_60ce9836f585a7205184.png with policy now +2023-07-12 12:32:51,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,066 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_7a4d6b6bbbebe28cdf93.png with policy now +2023-07-12 12:32:51,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:32:51,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:32:51,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:32:51,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:32:51,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:32:51,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:32:51,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:32:51,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:32:51,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:32:51,392 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0xlupm4k-media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:32:51,394 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ja8apt89-media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:32:51,395 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pba6jtoq-media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:32:51,435 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sh4g5hvy-media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:32:51,807 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2h42v809-media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:32:51,845 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ar23nxuk-media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:32:51,904 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zwe5rveu-media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:32:52,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:52,181 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v03si8sk-media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:32:57,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:00,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:02,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:03,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:03,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:03,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:03,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:03,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:08,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:10,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:33:10,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:10,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:10,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:10,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:13,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:16,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:16,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:18,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:18,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:18,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:18,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:20,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:20,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:20,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:20,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:23,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:28,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:29,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:29,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:30,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:30,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:30,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:30,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:33,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:33,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:33,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:34,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:39,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:40,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:33:41,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:41,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:41,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:41,244 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:42,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:42,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:44,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:48,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:48,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:48,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:50,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:52,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:52,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:52,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:52,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:55,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:55,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:56,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:01,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:02,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:02,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:02,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:02,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:05,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:05,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:05,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:07,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:08,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:34:08,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:34:10,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:34:12,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:13,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:13,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:13,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:13,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:18,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:20,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:20,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:20,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:21,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:34:21,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:34:23,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:23,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:23,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:23,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:23,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:28,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:28,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:28,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:28,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:28,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:33,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:34,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:34:34,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:34:35,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:35,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:35,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:39,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:40,219 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:34:45,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:50,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:50,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:50,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:50,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:55,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:00,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:35:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:35:05,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:35:06,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:10,220 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:35:12,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:17,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:20,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:35:20,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:35:20,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:35:22,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:22,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_88582cfed2f573568e58.png with policy now +2023-07-12 12:35:22,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,940 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png with policy now +2023-07-12 12:35:22,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,950 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_400a50fac321ed4c2979.png with policy now +2023-07-12 12:35:22,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,961 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_72521f709e19fb5db039.png with policy now +2023-07-12 12:35:23,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:35:23,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:35:23,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:35:23,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:35:23,258 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8zbk6k31-media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:23,259 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gsj7r3vp-media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:23,269 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6we7hu00-media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:23,282 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zudjuajy-media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:26,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:35:26,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:35:26,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:26,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:26,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:26,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:26,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:35:27,235 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qbv86jh1-media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:27,239 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f39ne2cy-media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:27,255 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hanrdfub-media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:27,258 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vfkufde4-media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:28,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:33,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:35,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:35:35,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:35:35,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:35:39,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:40,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:35:45,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:45,351 DEBUG SenderThread:3148485 [sender.py:send():369] send: exit +2023-07-12 12:35:45,351 INFO SenderThread:3148485 [sender.py:send_exit():574] handling exit code: 1 +2023-07-12 12:35:45,352 INFO SenderThread:3148485 [sender.py:send_exit():576] handling runtime: 45156 +2023-07-12 12:35:45,354 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:35:45,355 INFO SenderThread:3148485 [sender.py:send_exit():582] send defer +2023-07-12 12:35:45,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,355 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 0 +2023-07-12 12:35:45,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,356 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 0 +2023-07-12 12:35:45,356 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 1 +2023-07-12 12:35:45,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,356 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 1 +2023-07-12 12:35:45,357 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,357 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 1 +2023-07-12 12:35:45,357 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 2 +2023-07-12 12:35:45,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,357 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 2 +2023-07-12 12:35:45,357 INFO HandlerThread:3148485 [system_monitor.py:finish():190] Stopping system monitor +2023-07-12 12:35:45,358 DEBUG SystemMonitor:3148485 [system_monitor.py:_start():166] Finished system metrics aggregation loop +2023-07-12 12:35:45,358 DEBUG SystemMonitor:3148485 [system_monitor.py:_start():170] Publishing last batch of metrics +2023-07-12 12:35:45,358 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined cpu monitor +2023-07-12 12:35:45,359 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined disk monitor +2023-07-12 12:35:45,497 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined gpu monitor +2023-07-12 12:35:45,498 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined memory monitor +2023-07-12 12:35:45,498 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined network monitor +2023-07-12 12:35:45,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,499 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 2 +2023-07-12 12:35:45,499 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 3 +2023-07-12 12:35:45,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:35:45,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,500 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 3 +2023-07-12 12:35:45,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,500 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 3 +2023-07-12 12:35:45,500 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 4 +2023-07-12 12:35:45,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,500 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 4 +2023-07-12 12:35:45,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,501 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 4 +2023-07-12 12:35:45,501 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 5 +2023-07-12 12:35:45,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,501 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 5 +2023-07-12 12:35:45,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: summary +2023-07-12 12:35:45,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:35:45,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,506 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 5 +2023-07-12 12:35:45,507 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 6 +2023-07-12 12:35:45,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,507 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 6 +2023-07-12 12:35:45,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,507 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 6 +2023-07-12 12:35:45,507 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 7 +2023-07-12 12:35:45,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:45,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,508 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 7 +2023-07-12 12:35:45,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,508 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 7 +2023-07-12 12:35:46,237 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 8 +2023-07-12 12:35:46,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:46,237 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 8 +2023-07-12 12:35:46,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:46,238 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 8 +2023-07-12 12:35:46,238 INFO SenderThread:3148485 [job_builder.py:build():232] Attempting to build job artifact +2023-07-12 12:35:46,240 INFO SenderThread:3148485 [job_builder.py:build():266] no source found +2023-07-12 12:35:46,240 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 9 +2023-07-12 12:35:46,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:46,240 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 9 +2023-07-12 12:35:46,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:46,241 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 9 +2023-07-12 12:35:46,241 INFO SenderThread:3148485 [dir_watcher.py:finish():359] shutting down directory watcher +2023-07-12 12:35:46,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:46,925 INFO SenderThread:3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:35:46,926 INFO SenderThread:3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:35:46,926 INFO SenderThread:3148485 [dir_watcher.py:finish():389] scan: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files +2023-07-12 12:35:46,927 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/config.yaml config.yaml +2023-07-12 12:35:46,927 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json wandb-summary.json +2023-07-12 12:35:46,928 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/requirements.txt requirements.txt +2023-07-12 12:35:46,934 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json wandb-metadata.json +2023-07-12 12:35:46,934 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log output.log +2023-07-12 12:35:46,941 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_0838dad447e339fd0fea.png media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 12:35:46,941 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_a8dfbe0ff96ea5050767.png media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_56c2e575b2b9d92db6f0.png media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_3f8ba95016c3d0286ac3.png media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_b1104143852d0cdcfdbc.png media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_a492b392b73e859b8a77.png media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_3ac29936b57bab3b60df.png media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_0394766c00f144b38aaf.png media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ae28d8db362e0aa9f573.png media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_616207d75a28a906c55b.png media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_2c0d6dfc2d246283392e.png media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ed7f0d8809d791b958b4.png media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_beffc069b9ded929df09.png media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_83474bbbae95371e4fe1.png media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_8ede4c40d45be37a3114.png media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_cb5f34fc925cd4802446.png media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_6c49012791de94816289.png media/images/validation_50_6c49012791de94816289.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_5fc1a953a335a81d0538.png media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_f43b6d21c133e2a6b003.png media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_8bd9326976928e17b3ca.png media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_6ae01c6a6a2bfa713e44.png media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_cfabe222535005b7c73d.png media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ddf1c39281d80912060e.png media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ed363dbde584f9cd8a96.png media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_09c22363614ab958c3bc.png media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_eba1f69e2b11ef8f44d3.png media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_d01dfe2d2aeaa1bec479.png media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_24254896e341a078f630.png media/images/validation_70_24254896e341a078f630.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_64b2c49cff2d363d20ad.png media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_f3e5374d59497f9be535.png media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_653670180f2536edf28e.png media/images/validation_80_653670180f2536edf28e.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_ca894d05d078201593b2.png media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_d4b9dc3d41c12fcb9fe7.png media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_21a56bd8f7d955ce07a3.png media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_8b813df4910f6e14fe9f.png media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_b276db68b1c2650b37f5.png media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_661b1a72c771caa0e001.png media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_85a79c24055b252b391d.png media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_e3b8328c3efd11dc0868.png media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_5f675bc00f07f97610c2.png media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_ed89fce5f94211880d08.png media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_519be1aabb2ce5adb552.png media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_0a38067c3b4b0763814d.png media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_57b921c4856dfbee5311.png media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_b4dbdc5959d71b1b0010.png media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_3aa26b32048f4df01e9f.png media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_18e3cfcb4e5db9c7981b.png media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_f3f35ebd04d7e55194ae.png media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_3e5c971e8f903ffb6755.png media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_b27cf42472a6e5022d7a.png media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_db9b8ff059ac75e60033.png media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_1846f2582dc6add0423d.png media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_20276ffa7780c15c8826.png media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_5f10192137f91e3d88e7.png media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_f11466fe48b33d6f9da9.png media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_8cf36f31bd00d9abfd34.png media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_ad78c61f0f7db6d206fd.png media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_da52d8afd9d0f2869f66.png media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_c4f6c02b99e6533a2eb2.png media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_843770c3f4c58f7b4f6c.png media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_897ca9ce40094218e736.png media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_a6aefafa2d783d4803ef.png media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_c063d5e9ec2283745c8b.png media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_542fde61a9a7764e7963.png media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_7e7561227c49152fdf02.png media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_249e7863a11a9d2249ff.png media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_582ff9cdb53e3daa2736.png media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_80a5f2920f4985e1b760.png media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_bcd830ba6ea2f7b921a8.png media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_9f041d8a799e7d60ca1c.png media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_2942fa312a4899f548b2.png media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_57486bf611e806418a3a.png media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_1689c770ac115a929d63.png media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_18d5cba2a1158df428aa.png media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_c50d65d1788fc664335f.png media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_2b5b9594c7937018b6be.png media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_50004396bfec8e96747d.png media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_a321ae427f3731921032.png media/images/validation_200_a321ae427f3731921032.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_009b86ce19fa976b3f5d.png media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_fe7b08ac2a5afbc11b18.png media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_1f72d7f053af939e5276.png media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_ade80344c97e522275e6.png media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_b7889716d369f8c47db2.png media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_30fb41bcb1ce089cb7f8.png media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_f581c132868ccb4db30b.png media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_1b6340fa1d30a762feb0.png media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_58119209c10f17b2d40d.png media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_7e556c2aaa57f2ce8afe.png media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_a1e3c8f3087f5e3012a5.png media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_16d4939d62b11379b933.png media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_9dcda9052cdd403f3897.png media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_d631c6d270c38f7631a9.png media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_5cfbeaed31a9337bdbad.png media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_18282123a377a69416e7.png media/images/validation_240_18282123a377a69416e7.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_98861fa9aa94e0bb9466.png media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_607b3eebed55e6abe830.png media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_96a516914a407fcfc504.png media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_7a35da35c250d174b078.png media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_8833dd1eeb120dd86be3.png media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_cc24f9252c2efc365a1d.png media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_9d432cf0ad1538825247.png media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c5ef4fbb8722ad5da91d.png media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_4f96826bcb36d428d2d7.png media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c2fb4e9073164765d619.png media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_3fe25e6178ac8d39607e.png media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_630c030e959854fa6506.png media/images/validation_270_630c030e959854fa6506.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_5145710341bd620359f3.png media/images/validation_270_5145710341bd620359f3.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_18e1dc0d33fcdedb1fc5.png media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_4a948e463af013031d1f.png media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_ef3aba43d9e8e71c9284.png media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c25f63eee5ef9870fbd4.png media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c142b41b461357b2de28.png media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_ff574f4d8f5ce6c5ff1a.png media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_b84e927620cd6b0cb2a8.png media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1c16b044a10bd74e136a.png media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1b1094343a18b1720c55.png media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_a2de729812e74d6ab884.png media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_872691b49d402d6a737f.png media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1dc75f8c8767180b4449.png media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1a544bb059e5c537308c.png media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_619cd7b6e4fcfa89810a.png media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_5a10ea9a10a25e57135c.png media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_43c929f6445607c5f8d2.png media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_01b83aa2726efc837f38.png media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_73ad6d9a88337e318469.png media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_a373a58713d055f4d0ec.png media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_71caeb46293b077a4cd4.png media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_1627810819d85c5e49c5.png media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_4559efd7cd62cbc0d7bd.png media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_fc7d6afb7198af162914.png media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_7fbcfe2f1f50d8f10ad6.png media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_103af67d972f2199c1fc.png media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_f1d20714dd011423e767.png media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_dc0f97f353b7e7ab6dca.png media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_457d6bff4c2a7ae5f48a.png media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_2a1bc7c87c68196530cf.png media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_22aac1153c97ec2693c5.png media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_e283ca771222ebaf7b13.png media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_bb4784ba59026990690d.png media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_1c59f118bd1aa7b351ef.png media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_26e2ebd54ecac5225fde.png media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_d5ce8065e09a898bbb73.png media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_1483311cc0a22899fd84.png media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_09d003a6833146016be6.png media/images/validation_360_09d003a6833146016be6.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_66cc1fe983e4ffdf1c5b.png media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_fa56e2608c00ceaca35e.png media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_0b641787504f5e185899.png media/images/validation_370_0b641787504f5e185899.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_8ad08c11ab8ad22829d8.png media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_8126eeaacd234695183b.png media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_e3c3849a148163ace395.png media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_150783a4c31c17364129.png media/images/validation_380_150783a4c31c17364129.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_cf0b270e00fd03447bc3.png media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_4f13ece0060ef305e7c1.png media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_cece94bd69f475704bfa.png media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_b80214c7e7fafa67c5b0.png media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_dea73b102aef933ebfb7.png media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_b8bc388d0a6aa1ae82dc.png media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_301c182c9a0503b9c552.png media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_d5487564dfd56ce35ceb.png media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_5cbb16c19b007f3c49d3.png media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_36160e0e12a477c07ef9.png media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_975e274aed0125eac405.png media/images/validation_410_975e274aed0125eac405.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_c88421f1a33201c27e7f.png media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_3032692fd49e0f4ffa6a.png media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_2a287e605f741cc4ba84.png media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_f17cfd3caf204c385f2c.png media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_b5f6b2004b8fac7ef053.png media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_e80fbc111f2744e80e0e.png media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_582c2e8324c4a214eee2.png media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_31a8bb8062237a4e5e03.png media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_cc52ea02c073b4adb746.png media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_466a9d653c954a92a2d2.png media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_996fc8410c2def6b1ad3.png media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_b7c3984877608e60dcc5.png media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_37feadf7c6d79ab08bb8.png media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_850de8645d4e0ec1b327.png media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_62e3070332b5e7b50721.png media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_129489423a36c578b83b.png media/images/validation_450_129489423a36c578b83b.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_0be657c2999bb3f2f0e8.png media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_75c8cb1b49d83b55316d.png media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_a6c4a260b23627355952.png media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_69bab95618e258fac08d.png media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_de7722337cbd652fed8a.png media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_387823557d6d5cb784bb.png media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f98b8202065d26c1ef2f.png media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_ae66beb9ba6213202415.png media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_729f9b8d68182ccc9b8c.png media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f3357ea3af4ad6b0deea.png media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_57793f0e6246b307afce.png media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_36f12af806c3f1d50a42.png media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_f389fd00788c87466ea2.png media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_11e4b59458340be3e02c.png media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_e1df59c538a185bbc3ab.png media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_bb00c284bf371c7cee01.png media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_68abe5bccc0897c1b607.png media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_47a8d03f0cc699178621.png media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_03a01949302184444647.png media/images/validation_500_03a01949302184444647.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d74931567eda76fb139.png media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d16f7c02c5cc655df6a.png media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_89f5a8910a8c98de8cb9.png media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_3234686b4d18835995c9.png media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_da96661dc21e252cae64.png media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_34fac9bf4212cdd6d437.png media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_e3e834df071a8745bc59.png media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_61f1eebc64efc60d8ae6.png media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_8d7aa871f805bb466b02.png media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_95ea6a7d25f4eeb9af52.png media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_c82ca30fe9549b880c53.png media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_d9608828cb52fcf1ad77.png media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_a5b58d688f92b4fc1442.png media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_001908c5ae0df54582c9.png media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_9ebd350e55790857624d.png media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_978f41b570f2d5fd48e5.png media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_68ef6d897da63a408d01.png media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_37a3b637808816ae2a1f.png media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_fc4fe59f596766ac60b6.png media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_90a56ad8ada0c9c8274d.png media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_116ac864f0f13e57d81b.png media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_55055c73d3e5ab4513f3.png media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_402084574a959b5e4266.png media/images/validation_550_402084574a959b5e4266.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_607b86b7a89c36fdadfe.png media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a72665b3636d99570f6e.png media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a54da6547e3af36c2510.png media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_d160b3bcabae5e649acf.png media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_6d9eeb6998ef5daf6dc1.png media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_eb681e116684d158f76a.png media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_e92644ed701dc0b0faa2.png media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_9c89c3127ee7dbbf4a2a.png media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b26c685674fa1336fbab.png media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_fd8f16b41072ce3bceb4.png media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b60bdd2ea4c17924c6d0.png media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_d75cf57bdd0e29b1f7a8.png media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_a98fc70c70656fdfe63b.png media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_08863b59853c2669a10c.png media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_e05be95e710336b098d2.png media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_06bde4208de6769e990b.png media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_e5844c86288b2606b645.png media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_07de5acd372977760b4c.png media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_bbb685abfd63c223246e.png media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_6b5551cfcbdcbf8c81e5.png media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_1d981ecc323c30e20b81.png media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_b9cc12355deddaad9779.png media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_134e6d2a749fcacd43b7.png media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_92f0f0898c20a134b3c3.png media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_77093872b2a62f3184c0.png media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_f72e5adad3a7e2192d4a.png media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ed7326150a0c47d8bd9c.png media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ad1a36db317705c6c410.png media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_4f603f3934c8321cc8fb.png media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_79440b6bdd999b2e130a.png media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_da9c8b5a7202534ddcc5.png media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_bdab7dddcf554aad01f9.png media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_49779e88089791939e26.png media/images/validation_640_49779e88089791939e26.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_b64ccfa16d3aa44a2151.png media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_56616402a26a74e9ca6a.png media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_c1d1fb2c049800d964ec.png media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_7416b90b79fecff5e4ca.png media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_606f3fe637934aa6a15b.png media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_2b23cf43dc0d03af66c6.png media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_774f3994c62fdf69ce65.png media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_aca4a003a8d6b3b9f87b.png media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_4155d6ba117c33d5628b.png media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_65494724a21e7695e782.png media/images/validation_660_65494724a21e7695e782.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_3806edfc6615feb4c502.png media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_00b528cb36f02641e0c4.png media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_4b3b71fb783337ee170d.png media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_a54a1c21b8282d62f8f9.png media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_38a8a2cd9ee27689f5ea.png media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_7d58abcefc10987ef6e0.png media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_96b5253932542ff8e0ec.png media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_5420d62f64c76dfb3aaf.png media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_ee4486348a50df1eb1b0.png media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_1ff9e60aee819ba10a7e.png media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_8220342bb1673b65f66a.png media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_9749b628dfc772a6cc47.png media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_dd5fdf92beda60fe8a12.png media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_002747928010272ad4e0.png media/images/validation_700_002747928010272ad4e0.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_f85b220903af16d48ab4.png media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_ccf6b7bd526cf61a21d8.png media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_11469b6c8fd1d388b8bd.png media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_f8ad001e252a38c302da.png media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_e29ac85551bb1427b956.png media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_4e2183818f3e746eee9b.png media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_42e1f742968073010893.png media/images/validation_710_42e1f742968073010893.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_0ecdc87dea1892df4eaf.png media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_9c940486a7ed24710766.png media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_a812b1f440f7f52d6145.png media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_dc11c1be8869686a04d9.png media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_61523736602b93efd4f1.png media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_b3042d96cd6c0051282b.png media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_092236683fae816d1e06.png media/images/validation_730_092236683fae816d1e06.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_0d352eba665388c33ae3.png media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_4ca01dfc0ecd320c19a8.png media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_472a400ca408d21e03bd.png media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_d97239257eb1f6f99304.png media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_46e887ae6b4e77b77e34.png media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_6a4042dd0f145a5a90ed.png media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_d0782df35bd2b2c3cded.png media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_f007accf327156196715.png media/images/validation_750_f007accf327156196715.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_952512d46b1737ec3172.png media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_34b823a11a0cf4013af4.png media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_af5422b14a97cc645625.png media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_95d5c0a5c24789a3a475.png media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_00dc8fdce08e1ae13772.png media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_df68bb8ae7cb3ad86921.png media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_cba209758498588b8517.png media/images/validation_770_cba209758498588b8517.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_217409d7c0158fc1dcec.png media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_5bc6ddbcab2869abe354.png media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cd3d57f7ce32e8a2128.png media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cb1b948d1d2b956831d.png media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_6240b2c0fb96b2b24cb6.png media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_f18cdbdebab2bd2197d4.png media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_7c032ae6c9675c498efb.png media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_0e3bece4b24d9f0cff63.png media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_2c76661900a84617ca31.png media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_a66b82318049cd1e171c.png media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_ab1fea6190aed6bcea40.png media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_c3dd46dd047d49388d6c.png media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_9c63a115f12e85651901.png media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_9a62636574ba68d8fc83.png media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_8d25802cfd68926c4540.png media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_14a8076e3481bc7a8090.png media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_3d586403ca68c8dd89ef.png media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_9eeeace7edf859719f14.png media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_498f985682f98f0b515d.png media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_c37a81321646db9654fb.png media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_de63bd88f0f3c4502afa.png media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_5d081a2a0528a30e8ce2.png media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_0aca2d7047838c676be5.png media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_3bdbbfd9d4a0076d30db.png media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_73577e16f9e832867bc6.png media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_90e1e0a7b7d300559299.png media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_3cc72b352f6a24b1d731.png media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_924f6a01834fef3e5df9.png media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_8376c90a668eab0df0f1.png media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_d8f23be87659ed0cbfb4.png media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20f5209698af0ab0b778.png media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20c9060ffc3bdad4a934.png media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_1f72867e944a10d3e520.png media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_97fe22a2e02d107e0669.png media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_8062bd4106d50ccb836d.png media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_578c282df31c9a089068.png media/images/validation_860_578c282df31c9a089068.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_1ff63456305716e15a0f.png media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_b6004993162c340948e4.png media/images/validation_870_b6004993162c340948e4.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_1e7e11e8737344ca3c57.png media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_4f7e5756a84f41b58f03.png media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_674d99e976453deac9a0.png media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_c05827ace7407dafe67f.png media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_ddb13591ebb5fc4093e5.png media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_1a93cee3c2fe9238634b.png media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_8a9698cc7d3b8724fb2b.png media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_f18e48cd66c93df800cc.png media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_5f8815ecaca9bc2b8ad7.png media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_01b5da4e3887f97a0716.png media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_e9f5b2cb5a42a51a191f.png media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_6956b8efd11196c8a652.png media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_5741bd49ef7880023ce2.png media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_54eb3287687fb6f3d07f.png media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_4bf5e195f1228d67ca30.png media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_f56c6743f3df48566c7e.png media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_85b1581a7259fb0cc926.png media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_4bf762954166a9bf2bce.png media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_2e5e7f759b91d9be266a.png media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_3ac6586c86738660d7b9.png media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_7f02fda774e0cc0ec92a.png media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_8544a6b986b0e7758e17.png media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_4cbb72cb2a336b2f4f2b.png media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_a2e24d69369ee9545e6c.png media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_98329b1eb9c39ec50100.png media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_14811891692332578df2.png media/images/validation_930_14811891692332578df2.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_708543de31a4012af277.png media/images/validation_930_708543de31a4012af277.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_1aa1dd5671e7761dc7f9.png media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_25baa72fa29a6b87a7e3.png media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_55fb249dfc64d037ba51.png media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_0f4ce489e03b7c6499a4.png media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_8e55f5be94ca16a527e8.png media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_cedfd2e3a8d728719314.png media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_b66230be0915df8b41e3.png media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_32af62f9d0e790c02608.png media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_a3b1bfcfc8aabde7212f.png media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_5c7c719c7e9e5e608018.png media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_fdf70bc0fef8dac9c1a0.png media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_67790e930adf85923341.png media/images/validation_960_67790e930adf85923341.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_4600d4c65eb0657bf15d.png media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_133f183508d881d8f1f8.png media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_959d30c344a53f583035.png media/images/validation_970_959d30c344a53f583035.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_783f685672728f7cab19.png media/images/validation_970_783f685672728f7cab19.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_5bce29b811f38d81b7a9.png media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4522b255d71fb495f911.png media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4a474756239679aa6533.png media/images/validation_980_4a474756239679aa6533.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4e23c9690e64cda40444.png media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_e555318c04df0d898c35.png media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_d4658b3a29548774d64f.png media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_dd615cbf6d5a7a8a896d.png media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_952fc214368e10b389b8.png media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_1a96794a04b7af41b34e.png media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_e0021fa68cabb943f49d.png media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_87434c7eeda4190bf8d8.png media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_bd2b2cab96e55845d6e1.png media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_813871e148b66856b8a5.png media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_f52e87f356214ba43ded.png media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_8213e8a29e4df4a0c573.png media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_1fac8c596692012a9a71.png media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f7760d16dbb61cdadc4b.png media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_bc519090f4c42a857cf5.png media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_d02fa8190f5853b0b648.png media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f59fa5400ab07ea1eeae.png media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_bfd7171975011562507f.png media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_fd568449fa00bd93527b.png media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_a95db61e150a9ffc4641.png media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_3856c65003790d9a56ab.png media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_15446d27b696fee5751e.png media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_9a1046e9395d322b203b.png media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_e6fd436066e140c8ca6d.png media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_341a0c23b11ea49f2577.png media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_55de70da5261f9290e40.png media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_f35c05f1d08c6308ff14.png media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_9bfabce8c467677fe54b.png media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_1daa5fdb77023a407299.png media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_197022e2780b05209f12.png media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_06ba5b400f585cfe22b9.png media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_7be7ad3a7a700e0a150c.png media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_23fba533f1fd02e857f8.png media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_32f9f6540dbdc7c48f81.png media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_68b196a64c6e776ccd9d.png media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_bd0d7850e520c4cb0100.png media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_a070795b58d5c596307b.png media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_1f6acd9fcf45ba3ea19f.png media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_4817a2b823e0b176afd7.png media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_66da8376f918e5f31cac.png media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_6b2cf11ab85982d34290.png media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b8740b576006b9ee3546.png media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_76a3173208a51ee8c312.png media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_057775edd61cac3840c0.png media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b585e13abb070740f049.png media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_b9ea83bc203cb55d812b.png media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_ec943a3b531d5c037829.png media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_61651029309c609a7521.png media/images/validation_1100_61651029309c609a7521.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_1ad4a61cbb8909fcb78e.png media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_28d2a849740383813cb6.png media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_b889d46b029a022ecfc9.png media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_54b9949ee470b53f0543.png media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_eb513f39f1c260344506.png media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_3bb3ea14d134cd57deb1.png media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_2c6f306607bc7b9bfd92.png media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_8c2839a1d3140c3a7dfe.png media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_fd6b36b89f1e7fd2e78a.png media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_93e81f1f307564b83df8.png media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_331aea85c432690f9c0e.png media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_44a8006c0888f7a9b714.png media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_50f80b0ec1f9f7e17923.png media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_1961fdba2d4312886e8c.png media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_49f1b6dda25d69bd1eec.png media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_a158acb6a30a6a4fcc09.png media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_469091e483586cec37b1.png media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_9d89ef355de68c0d9ddd.png media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_bc32f73444fde5093291.png media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_c9a41cef5442cfbcf575.png media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_4646158f3cc2ef18d62e.png media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_a90fd9580e1c1e24f448.png media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6b84704eb15e35e5f9a9.png media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6e5ebba16091d67e0c49.png media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_9a7e5eb802701258688c.png media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_a451b10b1cddeff35521.png media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_2bc25132345cfabe28d9.png media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_f9f3ae84aa2fd03f0c14.png media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_e76ace79b32cf06e3105.png media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_d085f51cf12dae1345cd.png media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_f8627735c220e9667e1c.png media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_20948b93a6c00143c135.png media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_7b2ab005b9837744864c.png media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_f7abacf0c65108ebd291.png media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_c631ebe8975b90c43163.png media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_138faee5a37f54662918.png media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_597e7e8766e45fd365d0.png media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_707f52b23811c66eb81f.png media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_095c7719d5d88d3a784e.png media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_509f77863e0d740b58c5.png media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_c158e5c7fd160b30b673.png media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_1ba924a90889d370d938.png media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_25ef2dfcdcaf9ff09dea.png media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_a311e64f75624628145d.png media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_75f318a5faeb28d48e50.png media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_1015b192327aea438f8a.png media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_b37517440f90e43b7657.png media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_65b3c4bdf334afcfec36.png media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_624173894a8202fd049b.png media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_bed029a53fc783aa1494.png media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_82f6db57a6a62b37c416.png media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_0a392241e2b0e5c56de0.png media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_fe7e092e619d5a60940f.png media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9038ef38623aaf3b1297.png media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_4fdefa2382f1383aa03b.png media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9b87b6c7ef2509f2091e.png media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_563721340a1b7779ebb1.png media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_a92796d341fe8362362e.png media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_7e96d8ec566f76718aea.png media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_0ffe8cb158ff46492db0.png media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_4abee66cab5fc4486094.png media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_e893310e8c740aedacd9.png media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_3930a2a71b1d2e79ebf6.png media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_709f8444e3a90e20584a.png media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_52dd5c4bac63f8392a24.png media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_39451786e3d6986a15e5.png media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_7507600afa47a7721bc0.png media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_239d53a80d648e3b5d74.png media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_007f79d4a566d7c33ea0.png media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_91a32c15ebfc912c2201.png media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_76c909633b7c82a1c57d.png media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_d09f421be121a991d346.png media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_f520f48ddc3348a6c6bd.png media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_c67265e3a1dfdfb99387.png media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_e8b496a8b73ca169dce8.png media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_ad7f6abdff5cb1890622.png media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_499d8794ae851f2ca7c3.png media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_6d25eb3589d810ed3440.png media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_295181c2c8f74bdb7122.png media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_d49ed1edb101f8e16c80.png media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_47d0444bbe383a2f3ed2.png media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_5c747b5d08a35792803f.png media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_8d7535a093e8d31d2071.png media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_b02cf0aa477e22c8930d.png media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_59c1d57a32cc82a9a3ff.png media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_e5cb375d6da828cc3e3e.png media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_064f388777b3203b13c5.png media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_a24af0c79adf71caef03.png media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_9bd0004be4c4a0c4f009.png media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_b1042365de64e0a942b0.png media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_dbe013ae177630d5a987.png media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_3bca8acf12b4c46ab3ce.png media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_473197dd1f22461890d6.png media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_87ea3e96f26f5ebb57af.png media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ccdaf0fc6a17d6760008.png media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ff1f03141cde2a233297.png media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_a1fff07a952b497ecc10.png media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_f98823b83c4cee5e89ac.png media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8b4cb7fa6ec33b1868bc.png media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8fdfffa2228e3683ab99.png media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_e32477931d4fd98f823b.png media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_49ed245ee23ca275689e.png media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_7925401a6a5e9671ca9b.png media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_e9a621d16e71d8a70ce6.png media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_fea5b00a54b6cde2ffad.png media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_482ad8df2ce841450721.png media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_4e545b1d5aa48bae02e7.png media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_d6a886a91a9a1c97ff8e.png media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_e377b1a6785b8362f8b8.png media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_ba0b7fdc45db85580b1b.png media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_d46d1a613eb099818463.png media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_18b8edd8e82489c2e700.png media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_b7ab393ffe13b943540d.png media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_2048ef7db009bfd7c05d.png media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_02fa9f2ed8ff12c99aa7.png media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_fb6772a80e5d23777bc1.png media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_506d872797968a090e00.png media/images/validation_1390_506d872797968a090e00.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_66aa1d0173b44509cd1d.png media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_e84af5fa7376533300be.png media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_f29eae3956b8b680487b.png media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_d020640095c66cf43293.png media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_89c2655d6ea12bc678a5.png media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_754b2c1e14f253b4c743.png media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_0de4ce377ce0650003a1.png media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_57eaa364b09be7b1c708.png media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_65621df76009498b6856.png media/images/validation_1420_65621df76009498b6856.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_82f075e0330d5c29720e.png media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_8760dbb0b29df7f09a82.png media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_51314bed48a62a63179f.png media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_a5e480b9cae1eda40b25.png media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_6fd754942b22d95c6796.png media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_bb8a00b843066df3ae5d.png media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_acb50883d3668759985a.png media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_ff68423dd6be58cda77c.png media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_7456d4bbc7e1692f0395.png media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_2d1c628ad6cab3a92695.png media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_84ea7cea928542cee5c3.png media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_6eeb827ca7d660925ab1.png media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_1c96b81934ff3d0e0e7a.png media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_f51ce3f90deb8ec9156b.png media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_4fbcb05acbe8b16ae524.png media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_fc2c6a80029568453397.png media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_e0e86f6d8760c4a8cb7e.png media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_bafecf6d5c7a9080d983.png media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_688928104b18c14855ba.png media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_70617e686456e8dfd42a.png media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_724c86f275815432b7e4.png media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_841f2b0f1c0c1e17dd9d.png media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_d752e1cd90b5707c0547.png media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_a4d38678421736956450.png media/images/validation_1480_a4d38678421736956450.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_22921140211d309efc15.png media/images/validation_1480_22921140211d309efc15.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_577a6eadd1895fbf8edd.png media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_17a3ddb8f942d0a4a42c.png media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_0084cd73f87c0d0b983c.png media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_5443848005d4e6d0afaa.png media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_4b4b982230746456caa0.png media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_09e2e8132566f85bd196.png media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_c6e34c37f93334a75319.png media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_de9f5fe5746030c6a74f.png media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_cfb49806a7152804aa8b.png media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_9a03a9d55b0a9f41b234.png media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_be748917843477368009.png media/images/validation_1510_be748917843477368009.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_ec02c5d5a2c3f65267f6.png media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_d4a2e70a118507b473e4.png media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_11ba5556c102236aa637.png media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_3d2b9accd30d9956baeb.png media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_634e550898bfbdb9ac9f.png media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_790fbc2ddb08d6dd0d88.png media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_5140a0409a56d2f968a2.png media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_c983d04e3b22e1d9100a.png media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_91df6d540afc90aa229c.png media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_7e2daf917c565f387c73.png media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_6550992879b720b36137.png media/images/validation_1530_6550992879b720b36137.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_5f4cc7e816317d29edbf.png media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_a3ac4868653991969a01.png media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_d7dea15105ec789e1d21.png media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_cd95bda74fc7ee9c42fd.png media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_011beff0267deaa0300b.png media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_8efbafcfdf5d8df07263.png media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_1b7f50867d7f974e4323.png media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_bd10f02e7a1744e7844e.png media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_b236feda2a099cb72a65.png media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_503b3c4a7c702e3bde04.png media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_fb2e82a82bfa5264522f.png media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_137a9e1a20d097ef6ead.png media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_fb05536233b9d9384d99.png media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_8b73b1d4908d57e0f731.png media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_c996b3e0e81ceb767ce9.png media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_cdd5f325de677a2f831c.png media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_934a30ce9258abe3af2d.png media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_d7d4dbe10d69395e40ca.png media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_37f9cd55db6f0502d41d.png media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_cb784df1b13ebac5484a.png media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_d7d78a7a562011ba37c0.png media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_bada9f8f4e29cb81aa9e.png media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_da4a4411337b18959a1d.png media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_351faecc0b8489584c40.png media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_a9454bcbee87eab5aea9.png media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_9092d43f6e1c508c45c8.png media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_214ff49c4cb987612b43.png media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_aeea6b2da2c3bf58c7c3.png media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_1733cdf33e35398110c3.png media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_b6d946329d7752479070.png media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_f89b822e56af0bee16c1.png media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_49a77c36b783c7c3533f.png media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5f0c6d880a0a69fc8df8.png media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_cab2ce99009b26517a5a.png media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5e34d10c6734761c103e.png media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_0772971e314e9d07f59b.png media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_72968a408eafe4615fe4.png media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_de760783b75bdc640c99.png media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_bcdd70acdf0c52445fdc.png media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_661f798cc99bf53f5fc2.png media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_a26ecf3db23e2ccac681.png media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_585881b9e0f6d40e0ec3.png media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_68a5f846f45f697c2ead.png media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_64d777e8b185c0851910.png media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_f24a9e2785b102661151.png media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_dda90c1223ed981d3b1a.png media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_c45fc62adfcc71af96fa.png media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_3e49f286e7ccf53fddc7.png media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_000cc75e10c8a7929ff7.png media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_bcf133d130695d2171a4.png media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_9e45cdde0ed95786481b.png media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_d35268c957b3894ad085.png media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_c0361568b66056400dc2.png media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f129804ddaa9a57cd45c.png media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_4f3ac5bc86e6378f1a12.png media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f237ce678dbe9ec09de4.png media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_731d1b60253f6f4063e4.png media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_f040c34685a98d0f3850.png media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_8d56acd02a2f045be997.png media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_80603d23a803515f19ef.png media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_7295168e3d00a0e3d439.png media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_fbcc4f86c9d83ede1f5d.png media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_c1341754ea0507193be5.png media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f5c67f9540d8b5ed5c7c.png media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f31d3c5e28119395907e.png media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_508c64e8a2ff4f92e95d.png media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_46dc3e4359141341c6d4.png media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_bbaae7ee7d09d30b3e1a.png media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_92fe9627890fe0330f34.png media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_be9cd4b95a81eb492abf.png media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_603a92e14eb64a33c047.png media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_80be20563f1620981e46.png media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_2038713cb0876e29c6c8.png media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_95d7597f938efa3e944e.png media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_551525fb129586cd0c60.png media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_700fef9298b995bd049b.png media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_9fb3356673dd42522db4.png media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_a001a16142d0cdf3dd8e.png media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_62bcdb68c12bb4eee832.png media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_4dc54de8f062b7449876.png media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_9ead6b955150561d6d4c.png media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_d91846de1cf74e316293.png media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_bdda721ad2fd6569a964.png media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_1d177e30435b1c654b3d.png media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_06dd5b28c3be7a9bbdb9.png media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_94c6bfdf23b465d2a61f.png media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_88b7298825f502e5a062.png media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_0d6a954f28a281ccb3e6.png media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_e4640ecdc8c22558e54b.png media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_c9724817961bea50e2eb.png media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f3ed24daee483f81fe21.png media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_6f1689c0e9f7dc8526d4.png media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f03b061475b754e24cb5.png media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_e0be2e3dcde4725c11f7.png media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_43277f4f939e843ab8c9.png media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_437f85ccbcb8672d8508.png media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_19d50aae0838de593b50.png media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_1fbb3840c9621213d4a8.png media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_6c8d3f0f2e6689c4c999.png media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_73398702cb61fca4ba23.png media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_86559286380435a179bf.png media/images/validation_1790_86559286380435a179bf.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_1b5a6f042a0a4ba4a34a.png media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_3003f5b283d5c2f3d3a3.png media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_a5df1ab61da66e19d024.png media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_c6b5b46f40b7e3cc06e5.png media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_1d8ecdbc961ca85f19f9.png media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_22b0716ab99e4f2c1baf.png media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_db05126b5d482e1f86c1.png media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_e9390a064f93495e881d.png media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_0da500ef601e86506941.png media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_7ecf7da22bc3b3fad59d.png media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_3619de4ba8713b9bd91c.png media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_953f6fe5b812a7050a46.png media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b892c3791e1a5536a80.png media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_3cff7354bef942c8de30.png media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b9c6d907af1a36abe9c.png media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_d2e0fe71542ddbda5921.png media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_187a1a1f0c43344a0712.png media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_31e70925ba342cb958b8.png media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_2f7e6e14419630be9783.png media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_b75b67c6b0d3f3cd134b.png media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_82f6b3c68a48fade34f5.png media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_3a3b446f556528dde605.png media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_ec640dd671161721e4f7.png media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_c776faecdec7dde7f595.png media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_7b660814a00507637354.png media/images/validation_1860_7b660814a00507637354.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2bee3f76993cc3a6ba8f.png media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2ec1726523e7df6ef87a.png media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_763a0499812940f346e6.png media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_c031f9ce29973bad20f5.png media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_386744990d75b0d3adfe.png media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_96924703d0889517e752.png media/images/validation_1870_96924703d0889517e752.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_b0e70d9482eb144ce2cd.png media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_cf1d81e3b206f0eeddf8.png media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_e42387963e6467795ac3.png media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_a24606b4de934e6b61c1.png media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_19c60f744dd634323456.png media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_5fabc2eb5b38fc6c441f.png media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_362eab872b8e6854bfcb.png media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_abe953372839c2e35d11.png media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_61843d74c0b1b4039c5a.png media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_755c75927be8fb0bccfb.png media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_7ac136e8ccfb72e61b40.png media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_e8722f8bf9b351a33b0d.png media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_1895be2b314f5c3cc922.png media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_6fd922b416ffd009954a.png media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_5760a804d5e3912bf871.png media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_34349e41d8bc032e1c7d.png media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_f7431e028249d68b3d9b.png media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_0c56e18802b80a5e1cdb.png media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_7d921cd7daa3b8f8aae7.png media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_60d58608f7bfc9ddde76.png media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_3e7c1f6543b9b9815c1e.png media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_48056833d0b2d7f2157c.png media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_dc6aa83bfa1de0638963.png media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_8bf913d97ecc12b8092b.png media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_cbdfa1d5c7889c908498.png media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_55fed6437ec58a4992ec.png media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_b7870fe1d164423912ca.png media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_36d3d88a96d60bb7d69e.png media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_42b89bc7a5b6ad127dd2.png media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_2c29e26c44ad93206dd5.png media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1d43d5ef09db63ec9f2.png media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1baa48379524c1c8b17.png media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_e23c84b831732e39304c.png media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e3afb0220a3fb4fd12f3.png media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_35975d12907982332b52.png media/images/validation_1960_35975d12907982332b52.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e84cb7c3e576b3bd6e96.png media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e96fba72991bb49f94d6.png media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_1c5cd897c6da1cf72c60.png media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_b693a85e0ba7b021e8b5.png media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_2418660f78d4aba7393a.png media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_f777a51b498e8fce06d9.png media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_1db1fd51970584b5daa1.png media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_7a8ef7854cf8e37f9cf7.png media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_8caa5ac810c363d03b6e.png media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_647b73db1b8d573d85d7.png media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_e53b48fcc568a502f437.png media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_7e8731b3e3747b303dfe.png media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_283a622f33c91b0b5af0.png media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_35002cfdf6c8ccfffdcd.png media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_1f7c7153c03240078b4d.png media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_56c478641cd971b63c78.png media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_0669630fb67332c6ded5.png media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_81cc3487bd6c41e95ef1.png media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_ebb2fde791af714a87fa.png media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_fe06a780b0eeb017b688.png media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3246141969b6e42324be.png media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3fcb388422ea2886399f.png media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_13af1ccf733709899b51.png media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_6f7e3605df1d5a3bcd5d.png media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_ae08a04cbc13fea1f1e4.png media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_05878893911aa97a5d5b.png media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_1821049762b6b5d37c08.png media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_e34ade0521fc9adbaae3.png media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_d0209e06d5c81f80ec0d.png media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_835fcd4e7ec0584630ce.png media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6a4be91a47a6698809a9.png media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_167355b47a08bf8109e9.png media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_7914a4c603a50f2ea240.png media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6b091a76f28e18553ef9.png media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_95ba8564905e15131c15.png media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_025a92e01716e1b46d6d.png media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_d4fb051468326f3eb57a.png media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_8ea42d44ae0516e0ca5e.png media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_e0fa444f603a66029fbe.png media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_d6f502d4c0d1394b0881.png media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_ad32613282f23549a8bd.png media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_5f6fadce63310a783572.png media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_19ed657f9c10dbd875ca.png media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_e5f36a2a698bd96464b3.png media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_cf0e421ac16cf53c2d5a.png media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_df0b7fe6b5a5f888eeac.png media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_df994cef7f6357e3664e.png media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_4c3033cd3536b3f44094.png media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_0d50a284079de0683ad8.png media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_7ed8ebb926937feea5e8.png media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_acf992ec2207b3ea2d60.png media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_f680c2bff58b85913aa3.png media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_b72a6b1b7e4871bb5d8f.png media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_c267c69f2a522dc95745.png media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_ed6b49b703996f58696a.png media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_333e872b5de120d26892.png media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_38040cf3cd9cbe1bc314.png media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_2c125071cfc5a2fe04e6.png media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_62fae4a01ba8422d6ed9.png media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_008df6c26e19a483defd.png media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_3ef95d0764b88f6fbafa.png media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_6a7c69b67225aaafd76d.png media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_b4413d53814dba669bcf.png media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_7999661823c5d1a194ca.png media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_1d395d84a1aa765d6c1b.png media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_fe5c8ad23d0df9553dbf.png media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_1b4ddf79dc604699f630.png media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_259ec0830802f8053fe5.png media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_bb1524320261cb0216db.png media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_0099dae297478bd3ce67.png media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_825c97191445f88d10ad.png media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_ebe038f09b0a2e5c2779.png media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_69c0860f204684fe6617.png media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_a9fe0e5593aba42cc7e6.png media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_46dad1ead66f697f0698.png media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f451e47ddafe0bf1fcc6.png media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f44d5fe284ffc3b74829.png media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_4f03c9db92527c876ba9.png media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_f78f21bafff0abaf0089.png media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_1ee652c4e539d1e07cce.png media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_b97ae163d55862639f70.png media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_ec40a3df133e538d57f3.png media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_36b4d3a784d14fee51a0.png media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_33d957838c074e90f6a2.png media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_533582fdbffc2ddc97ac.png media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_def8ab43612eed95556d.png media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_d63b39c78c18b127efec.png media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f1a0da8e451df830c88e.png media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_5d7322114061dc65484d.png media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f5a9c542fcce2ab79bf5.png media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_b89f7b812f07101e7bc5.png media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_51f75fba279dc7d2d626.png media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd64ca37330df964f268.png media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd935f0213029522bb6c.png media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_963ea35b0f8a3f28bf7f.png media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_5479be16d55799087c1d.png media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_485475519d4a6c767788.png media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_8e5e62fe39fe252d734c.png media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_3d504d65d8985e43c4df.png media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_7e6627623494cc0c7f15.png media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_14e783bea121b92109bd.png media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_b4ca94832829e41b0eaa.png media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_fd2b6676675f447dd0bb.png media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_a2b2236edc7a209199e4.png media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_be095f7407792a1d8c2b.png media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_7995e8c15e47416a5a98.png media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_fa6aae454ea563a1568c.png media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_bba8aa2011cbb2b9e998.png media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_cc1f25e32b609b2e7f13.png media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_c694e9556fc95a54794c.png media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_c76d157bbb397161c214.png media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_cbb566c39078908ba8b3.png media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_17e81b81f9d5ecefe6ad.png media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_77e784d7b8aacf9759c6.png media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ddcd1a1e8b54f46f7b26.png media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_c1f37bde1c536851ff31.png media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_84e331f8cb88258072e5.png media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_1d09dab1e10018c0f5a2.png media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f5085a2630d7643ccb37.png media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_ea013c7b85e22fd83c96.png media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f2fc182f190a6e4d3266.png media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_57fbdf8438f8224664e4.png media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_66b7fc2a071b6e92e172.png media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_1db6dc00cd88a44aa4f5.png media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_506c3fdf835cc8b9e9bc.png media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_d6d98a9fc2b412d909b2.png media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_9e0da1558268d346b2ff.png media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_a95379a0e1bb4ee5eefd.png media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_50d6f6aebffcec48f478.png media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_dc4de63f9960a0a31886.png media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_d8ebb18636c7304a48a5.png media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_8ab18a5e3a00fe86d6ef.png media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_f6c9e9af5b280aff421b.png media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_229851204683a32b894d.png media/images/validation_2300_229851204683a32b894d.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_b9647059d362b9b7c491.png media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_abce74fb7ce87882e4ef.png media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0b62299b4392d7dab3b5.png media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_8adc40877878e2c5992a.png media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_b8447334c2fa99557dab.png media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0a4470d06e9e5634fc30.png media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_6e0bdeefa42f0abef07e.png media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_cad2712e0579795eae61.png media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_8982b4ca24e18e3a2e66.png media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_d536893289030f727428.png media/images/validation_2320_d536893289030f727428.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_9d435e637b3005c16f38.png media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_c9b9321adf648747a0cb.png media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_86f3d365c9fc4e6ab5e8.png media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_1afc2708fa415a3e0f40.png media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_e1920e85b34fdfb6609a.png media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_65d2e2217439693f9468.png media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_cfd6e181a1bb997a54d6.png media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_d7a930a7a71d94c1211f.png media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_576ed58f083c4925ff4f.png media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_6b91f2effcbccf59aafb.png media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_7116b69ba382286251a6.png media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_4776174cbccf4dcad7cf.png media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_3d170b31330953e821ba.png media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_6f1a9a30ce0054e1f213.png media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_a603ae3e847019132432.png media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_4eb285dbb680a44d3c77.png media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_6b60edf29451c64b6e2b.png media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_e3cc962dadab1208b51d.png media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_4fd19c0f067294c7a9e6.png media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_81faade70bf66d35a629.png media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_841e12bb766b52954d05.png media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_3b3663922cc96b138671.png media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_f14220ee5227d3539481.png media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_b3144dac275fe828dece.png media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_34c10cf3cd6d89963a05.png media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_92717b0771dd961acad8.png media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_674a3618e81973b0470a.png media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_0c825ba6ed95b9246d26.png media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_7fcaf32b847afe6bc2ca.png media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_3cef30f8c2574b5c86c0.png media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_a254fe937a4ee82dda50.png media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_f055f10be828e22bd53d.png media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_fcac0499a5c4fff7482f.png media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5eb323fa89395b0ed41c.png media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5870baca82c8f1ea9c65.png media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_e300a087119470bd19f7.png media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_a3ef41aa53f7542e2ff6.png media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_86a10e9e981b83c54181.png media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_508d57a7914d9268a798.png media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_d23160faa6354332ac90.png media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_e3cbf88a5d454b013c98.png media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_bdfc2304f95d55e3c519.png media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_9d7fc07bee22787a2007.png media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_f15d4e86eefcf39e1eae.png media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_8826c1292b54e84dc27b.png media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_02cad4f86332c4d30e0f.png media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_b90423e76bb46ed0bd51.png media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_f4dd09917aefe94149cf.png media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_807425807eddcc0e0aaa.png media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_30096936a6475e462a39.png media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_8fe94ed8c310dae4cd3c.png media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_d5445284771d7b2c66a0.png media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_c867e6da32e13d963e59.png media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_815ded56ecd664726052.png media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_e5fd4b41b8c0c33c93ac.png media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_edd32e44eaf81373d115.png media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_db2b2e82dd30d35d1311.png media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_8dcd67f2db2d9902e40b.png media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_97ab1f90f62f7fd81135.png media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_b53f3dded240b500b1cb.png media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_a061712ad948dbaccbd5.png media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_129f224b0015451a7862.png media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_6c96769927607cfa02a7.png media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_e2238e57e86ad9eb1236.png media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fbe171c1140c25cd1cb2.png media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_eb52c1f3852b2db713d6.png media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fb4911582a74ff6e59f1.png media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_1a8029b03d2f4e8ba6b5.png media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_a3765156a53105d7a44f.png media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_7b6d1552c5731ce4d99a.png media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_45f174c3353c67e7d2fa.png media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_f73a3d21c0ad929cc0df.png media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_2345ba38c318660ea20d.png media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_4acac46bf9cdc8077cd8.png media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_ce5f5d922b12cb9a8e58.png media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_55683596f533718ecbc7.png media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_9cd53a6492dff1119fcd.png media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_4e5b7eac0474303b7db6.png media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_a5bdcb2b01b23f824fcf.png media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_e419bc8806d8bdc57fa0.png media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_48e39b87d59cc06724b3.png media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_4436a3e5cc352d2a1017.png media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_03d4142985e8dc5f61fc.png media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_13c9cc17dc94ed86351c.png media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_97a6019e1743b1256999.png media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_1248be5f76e40ac7db9c.png media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_8cdd64e4abd06459cc31.png media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_5f9749e51f806c3a7042.png media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_b1d38d2764a5721d15ac.png media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_7e913201741ac8370fe9.png media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_35fe8e67bdbd375bad51.png media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_e4060fd5427ae0ecee26.png media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8ef00575ef619617eb8f.png media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8c7622cbbe1b8f2266c2.png media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_5dcd5f49c5610ed98e57.png media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_ceacc8064824cf24b9c3.png media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_13fad504fb5c2ac0ee04.png media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_0287eb8f3f80591bc234.png media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_784615f77a2b4e7047a9.png media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_38ab10654e678fd3808d.png media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_0362f6603354bb565423.png media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_12a798eb5ac3d54ffad6.png media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_62ad7e090c150c4642cd.png media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_2342de4c23f89a71e70a.png media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_12a6034cf8cfc23b5df1.png media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_59ef69c8ce6d4ab2107c.png media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_404750014d2070b4b00e.png media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_e5c153d3981bac3820e6.png media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_305e21c628b26bb7d55e.png media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_5996b8fa55b566dccbed.png media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_a3c7e4b8847949aa758f.png media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_d8ef17b50203820bbbe3.png media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_5b8b7b248eeeadd01dc0.png media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_3c27c596a729886e6078.png media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_8d036206f070328af8f6.png media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_b1cd6dc23fe6b08880be.png media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_792f17b1a307463180d2.png media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_87526415e87acfa56b82.png media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_af826553a0f3457c21a4.png media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_1e363e4fe52a8dd1f3d5.png media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_01eb7980170d3a100648.png media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_f752872a939e31b6a407.png media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_cbe3a2e2923fea324bbe.png media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_ece7f7211bafcdaa62b2.png media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_8852be70b2d4d0b9cda6.png media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_60a5b23b74be955f2c27.png media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_e1afc2c90dd83c53afe8.png media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_3a9f356aa83d5b8c3154.png media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_feb61af945632b2e5a0d.png media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_bd16c2dda6cb242546bd.png media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_39f7775ce6273f5abdd3.png media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_be22a91ba76075904920.png media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_570101fe2f3838813987.png media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_049933627fe17227a897.png media/images/validation_2660_049933627fe17227a897.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_b7c6a59e577b6aff9423.png media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_8f7a128a9aebab91d68e.png media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_abd6457914db5ecb11f5.png media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_81b157e520543297899b.png media/images/validation_2670_81b157e520543297899b.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_3ca3ef00974e8561788d.png media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_7cb40a3f835111c25bc8.png media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_1128a7fc997078f6857d.png media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_5968f013df449c836103.png media/images/validation_2680_5968f013df449c836103.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_f6facb6c2b52824b8087.png media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_ea076148ce2508581f4a.png media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_6b92b33ac5d9538cd14f.png media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_09839606b108aef4ba45.png media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_2a9c6e3e9705c484c43d.png media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_951a6312bd4ff445e3bf.png media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_cd74dc827e683d49ff49.png media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_9d44392c5a9c6cbb63a6.png media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_de3f2b25e35e4537c17e.png media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_5ca25be3dc62ed26aa0d.png media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_1328ec76d8379d97a378.png media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_6e2b6aa87197c3eba430.png media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_fb1e12ca3be46bca7950.png media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_cab768f5b1c4da57e828.png media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_9ebc2d1c275cb06de97e.png media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_62fa4b4be6b75191315d.png media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_468e29d1792ddfc31f96.png media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_ec678e94be72a14ad671.png media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_7a6b051b3cd7a36b4c2e.png media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_a1852227cbd29c30acc0.png media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_a1a975ef1b62e7916638.png media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_3a9de177b9a56588f2fd.png media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_88328b2a91d7dcdc342e.png media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_1eadc0d7d441ac97c181.png media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_809fb9a00d33011ef1b7.png media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_35d816078fba6fbc5f20.png media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_e2650e26371a6e674cfa.png media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_44058a9b1a42011478fa.png media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_70b32648987a29c582d5.png media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_aab6a0c7b6bb6712ffaa.png media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0eccab278b3c49ff65d0.png media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_a6f733cd68a255bc3547.png media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_6c225fe157884e440759.png media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b06c0ff3bb6bd189e084.png media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b32e150d333150b6d60d.png media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37df37da56d5d142423a.png media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_0ebabc283025333f43a1.png media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_5cdc5d44821692ab8816.png media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37cb82e9b9e53c9e5c5d.png media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_5a82d28324cd9b502db5.png media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_1df5c10fc9ddab2abd27.png media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_a3c09a1cdfb2790cb544.png media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_b0ea698af085bca141c6.png media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_0a4eb1c4607d5a5fd63e.png media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_a4a930c3aca9ed6b1e66.png media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_1e810046ab391ec21457.png media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_55e6ce392e83cd6c9775.png media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_79746b73a2d51ab398aa.png media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8fd491043a269feded86.png media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8e4329d82c9013ca58f7.png media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_1f22abcfa3a067c7928c.png media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_0393c6fc4db3e318d3d3.png media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_4fee48092f2ede4f3c25.png media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_ba69b7651ab0dcd73bdc.png media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_af052cd0179c44de7cb7.png media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_a5847a1c9ac576d3c242.png media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_d0edf809dce9067d913a.png media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_1176c8bf7fff463bf9c6.png media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_f3f42317c0356c47ff20.png media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_49d3966f62f54fe9ca62.png media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_a708be3afa6ae1a86e60.png media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_7e7433279443a3a2bfa6.png media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_2d3f82af7844062f4b35.png media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_dfe649466fc1947dac85.png media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_2975002e2263b7b13da6.png media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_462a59d1382cf36adaec.png media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_9c5803c959d6e590aeae.png media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_c18b3c294fb48283f9a6.png media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_e995c389d61465ef369b.png media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_a6d367b1ce47fd112b6d.png media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_556f808727f18f3fb7b6.png media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_bcac2792455fc0c610c6.png media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_0e8bce2d57e7e047c357.png media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_fe5c449ea1876a5e69ca.png media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_1d174f0fc051b4fa542a.png media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_9b8a7d4902a70bbe3529.png media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_948197ed3bdc8486cb6a.png media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_7ab6e23c2268ebda9ab0.png media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_ad77bbc22d61066c6ce9.png media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_cb1bd764507bb8c18d8a.png media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_8578482fbd741b1fda1e.png media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_98e0fea80621818b19b9.png media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_5371e29c2e2d3a7d3242.png media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_3d0ad7c57f2af82f8bbc.png media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_8f1fe1b72bfe04a0b014.png media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_aa4b86f6ab2a980e6525.png media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_27c9042e9f887913b3c0.png media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_2d10951a0f8b9aed5734.png media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_42b541158c5e9a2b90ee.png media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_ba216e11217e75e9719b.png media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_885722981f42772a687d.png media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_3f89f7c3f5ff7126de27.png media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_ec96bb36cf4e66588cd4.png media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_e0997462fe5eec3ca9d3.png media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_0a30f7284bacae8356b3.png media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_fd71982266285f250c3c.png media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_2268465f1770440e1c31.png media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_94f80d2b1ffdc85aabfa.png media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_ccf35b7fa382262ad056.png media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_11bcda97ccccef6067ba.png media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_ba511e69b1b4757bb114.png media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_32467ef9d42b5e1e1a33.png media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_c49f0237624dac615f66.png media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_5189a1941e0ee2300ecf.png media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_dce90c9fdac2309feb19.png media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_0d199df0c17d88c8726a.png media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_626d6d1dadf7aad42361.png media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_7b43af642a476832b780.png media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_76db5a33fad883eea4bc.png media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_366cee911e557f14b4e5.png media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_95e48c47882f1ab9ab69.png media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_c7e2d3867abfa6c74ec7.png media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_da274c6af8f1eb4795bd.png media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_3fff3d3522577450efef.png media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_949364012c78ad26092f.png media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_86a486a36860b5c6245c.png media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_18ea565e486f4bef2bbd.png media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_dfadefec7519a6555efa.png media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_26cc58965abb0300912c.png media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_d1566b7d6fc6fface55f.png media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_3375e10d6215e10e1662.png media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_60ce9836f585a7205184.png media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_7a4d6b6bbbebe28cdf93.png media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_88582cfed2f573568e58.png media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_400a50fac321ed4c2979.png media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_72521f709e19fb5db039.png media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 10 +2023-07-12 12:35:47,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:47,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:47,059 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 10 +2023-07-12 12:35:47,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:47,060 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 10 +2023-07-12 12:35:47,060 INFO SenderThread:3148485 [file_pusher.py:finish():159] shutting down file pusher +2023-07-12 12:35:47,340 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/config.yaml +2023-07-12 12:35:47,349 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/requirements.txt +2023-07-12 12:35:47,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:47,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:47,367 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:35:47,379 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:35:47,579 INFO Thread-11 :3148485 [sender.py:transition_state():602] send defer: 11 +2023-07-12 12:35:47,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:47,580 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 11 +2023-07-12 12:35:47,581 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:47,581 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 11 +2023-07-12 12:35:47,581 INFO SenderThread:3148485 [file_pusher.py:join():164] waiting for file pusher +2023-07-12 12:35:47,581 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 12 +2023-07-12 12:35:47,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:47,581 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 12 +2023-07-12 12:35:47,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:47,582 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 12 +2023-07-12 12:35:48,314 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 13 +2023-07-12 12:35:48,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:48,314 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 13 +2023-07-12 12:35:48,315 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:48,315 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 13 +2023-07-12 12:35:48,315 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 14 +2023-07-12 12:35:48,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:48,316 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 14 +2023-07-12 12:35:48,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: final +2023-07-12 12:35:48,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: footer +2023-07-12 12:35:48,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:48,317 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 14 +2023-07-12 12:35:48,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:48,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:48,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:48,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:48,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: server_info +2023-07-12 12:35:48,325 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: server_info +2023-07-12 12:35:48,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: get_summary +2023-07-12 12:35:48,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: sampled_history +2023-07-12 12:35:48,481 INFO MainThread:3148485 [wandb_run.py:_footer_history_summary_info():3467] rendering history +2023-07-12 12:35:48,481 INFO MainThread:3148485 [wandb_run.py:_footer_history_summary_info():3499] rendering summary +2023-07-12 12:35:48,482 INFO MainThread:3148485 [wandb_run.py:_footer_sync_info():3426] logging synced files +2023-07-12 12:35:48,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: shutdown +2023-07-12 12:35:48,482 INFO HandlerThread:3148485 [handler.py:finish():854] shutting down handler +2023-07-12 12:35:49,325 INFO WriterThread:3148485 [datastore.py:close():298] close: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb +2023-07-12 12:35:49,481 INFO SenderThread:3148485 [sender.py:finish():1526] shutting down sender +2023-07-12 12:35:49,481 INFO SenderThread:3148485 [file_pusher.py:finish():159] shutting down file pusher +2023-07-12 12:35:49,481 INFO SenderThread:3148485 [file_pusher.py:join():164] waiting for file pusher diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..02e5db1bef6bfe0a62a90637469ea3bf8bf52bdd --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,42 @@ +2023-07-12 00:03:08,079 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4 +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Configure stats pid to 3148323 +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'} +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/logs/debug.log +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/logs/debug-internal.log +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():547] calling init triggers +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():554] wandb.init called with sweep_config: {} +config: {} +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():596] starting backend +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():600] setting up manager +2023-07-12 00:03:08,084 INFO MainThread:3148323 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-07-12 00:03:08,087 INFO MainThread:3148323 [wandb_init.py:init():606] backend started and connected +2023-07-12 00:03:08,091 INFO MainThread:3148323 [wandb_init.py:init():703] updated telemetry +2023-07-12 00:03:08,092 INFO MainThread:3148323 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout +2023-07-12 00:03:08,319 INFO MainThread:3148323 [wandb_run.py:_on_init():2176] communicating current version +2023-07-12 00:03:08,412 INFO MainThread:3148323 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2023-07-12 00:03:08,412 INFO MainThread:3148323 [wandb_init.py:init():787] starting run threads in backend +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_console_start():2155] atexit reg +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_redirect():2075] Wrapping output streams. +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_redirect():2100] Redirects installed. +2023-07-12 00:03:08,573 INFO MainThread:3148323 [wandb_init.py:init():828] run started, returning control to user process +2023-07-12 00:03:18,853 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4 +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Configure stats pid to 3148323 +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'} +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_init.py:_log_setup():507] Logging user logs to /nfs/nfs2/home/sragas/demo/wandb/run-20230712_000318-e287bzfl/logs/debug.log +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_init.py:_log_setup():508] Logging internal logs to /nfs/nfs2/home/sragas/demo/wandb/run-20230712_000318-e287bzfl/logs/debug-internal.log +2023-07-12 00:03:18,855 INFO MainThread:3148323 [wandb_init.py:init():547] calling init triggers +2023-07-12 00:03:18,855 INFO MainThread:3148323 [wandb_init.py:init():554] wandb.init called with sweep_config: {} +config: {} +2023-07-12 00:03:18,855 INFO MainThread:3148323 [wandb_init.py:init():591] wandb.init() called when a run is still active +2023-07-12 00:03:18,856 INFO MainThread:3148323 [wandb_run.py:_config_callback():1283] config_cb None None {'pretrained_model_name_or_path': 'runwayml/stable-diffusion-v1-5', 'revision': None, 'dataset_name': None, 'dataset_config_name': None, 'train_data_dir': '/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/', 'image_column': 'image', 'caption_column': 'text', 'validation_prompt': 'tom cruise thumbs up', 'num_validation_images': 4, 'validation_epochs': 1, 'max_train_samples': None, 'output_dir': '/l/vision/v5/sragas/easel_ai/models_plain/', 'cache_dir': '/l/vision/v5/sragas/hf_models/', 'seed': 51, 'resolution': 512, 'center_crop': True, 'random_flip': True, 'train_batch_size': 2, 'num_train_epochs': 300, 'max_train_steps': 3000, 'gradient_accumulation_steps': 4, 'gradient_checkpointing': False, 'learning_rate': 1e-05, 'scale_lr': False, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'dataloader_num_workers': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'push_to_hub': True, 'hub_token': None, 'prediction_type': None, 'hub_model_id': 'person-thumbs-up-plain-lora', 'logging_dir': 'logs', 'mixed_precision': None, 'report_to': 'wandb', 'local_rank': 0, 'checkpointing_steps': 500, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'enable_xformers_memory_efficient_attention': False, 'noise_offset': 0, 'rank': 4} +2023-07-12 12:35:49,765 WARNING MsgRouterThr:3148323 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20230711_182238-06r7b4m3/files/config.yaml b/wandb/run-20230711_182238-06r7b4m3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1804210327cd80f7491fd8fd748f3cee8a3090f --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/files/config.yaml @@ -0,0 +1,39 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.8.10 + cli_version: 0.15.4 + framework: huggingface + huggingface_version: 4.30.2 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1689114158.539519 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 71 + - 83 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 71 + - 83 + 3: + - 23 + 4: 3.8.10 + 5: 0.15.4 + 6: 4.30.2 + 8: + - 2 + - 5 diff --git a/wandb/run-20230711_182238-06r7b4m3/files/output.log b/wandb/run-20230711_182238-06r7b4m3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8a89548f9612946eecb7154c6e4b3d2086a381a5 --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/files/output.log @@ -0,0 +1,7 @@ +07/11/2023 18:22:40 - INFO - __main__ - Distributed environment: MULTI_GPU Backend: nccl +Num processes: 4 +Process index: 1 +Local process index: 1 +Device: cuda:1 +Mixed precision type: fp16 +Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 44533.36it/s] diff --git a/wandb/run-20230711_182238-06r7b4m3/files/requirements.txt b/wandb/run-20230711_182238-06r7b4m3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..631710e9e888cc8a873b5d74d41732719a57dd54 --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/files/requirements.txt @@ -0,0 +1,165 @@ +absl-py==1.4.0 +accelerate==0.20.3 +aiohttp==3.8.4 +aiosignal==1.3.1 +anyio==3.7.0 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==21.3.0 +asttokens==2.2.1 +async-lru==2.0.2 +async-timeout==4.0.2 +attrs==23.1.0 +babel==2.12.1 +backcall==0.2.0 +beautifulsoup4==4.12.2 +bleach==6.0.0 +cachetools==5.3.1 +certifi==2023.5.7 +cffi==1.15.1 +charset-normalizer==3.1.0 +click==8.1.3 +cmake==3.26.4 +comm==0.1.3 +datasets==2.13.1 +debugpy==1.6.7 +decorator==5.1.1 +defusedxml==0.7.1 +diffusers==0.18.0.dev0 +dill==0.3.6 +docker-pycreds==0.4.0 +exceptiongroup==1.1.2 +executing==1.2.0 +fastjsonschema==2.17.1 +filelock==3.12.2 +frozenlist==1.3.3 +fsspec==2023.6.0 +ftfy==6.1.1 +gitdb==4.0.10 +gitpython==3.1.31 +google-auth-oauthlib==1.0.0 +google-auth==2.21.0 +grpcio==1.56.0 +huggingface-hub==0.15.1 +idna==3.4 +importlib-metadata==6.7.0 +importlib-resources==5.12.0 +ipykernel==6.24.0 +ipython==8.12.2 +jedi==0.18.2 +jinja2==3.1.2 +json5==0.9.14 +jsonschema==4.17.3 +jupyter-client==8.3.0 +jupyter-core==5.3.1 +jupyter-events==0.6.3 +jupyter-lsp==2.2.0 +jupyter-server-terminals==0.4.4 +jupyter-server==2.7.0 +jupyterlab-pygments==0.2.2 +jupyterlab-server==2.23.0 +jupyterlab==4.0.2 +lit==16.0.6 +markdown==3.4.3 +markupsafe==2.1.3 +matplotlib-inline==0.1.6 +mistune==3.0.1 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.14 +mypy-extensions==1.0.0 +nbclient==0.8.0 +nbconvert==7.6.0 +nbformat==5.9.0 +nest-asyncio==1.5.6 +networkx==3.1 +notebook-shim==0.2.3 +numpy==1.24.4 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +oauthlib==3.2.2 +overrides==7.3.1 +packaging==23.1 +pandas==2.0.3 +pandocfilters==1.5.0 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==10.0.0 +pip==20.0.2 +pkg-resources==0.0.0 +pkgutil-resolve-name==1.3.10 +platformdirs==3.8.0 +prometheus-client==0.17.0 +prompt-toolkit==3.0.39 +protobuf==4.23.3 +psutil==5.9.5 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==12.0.1 +pyasn1-modules==0.3.0 +pyasn1==0.5.0 +pycparser==2.21 +pygments==2.15.1 +pyre-extensions==0.0.29 +pyrsistent==0.19.3 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +pytz==2023.3 +pyyaml==6.0 +pyzmq==25.1.0 +regex==2023.6.3 +requests-oauthlib==1.3.1 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rsa==4.9 +safetensors==0.3.1 +send2trash==1.8.2 +sentry-sdk==1.27.0 +setproctitle==1.3.2 +setuptools==44.0.0 +six==1.16.0 +smmap==5.0.0 +sniffio==1.3.0 +soupsieve==2.4.1 +stack-data==0.6.2 +sympy==1.12 +tensorboard-data-server==0.7.1 +tensorboard==2.13.0 +terminado==0.17.1 +tinycss2==1.2.1 +tokenizers==0.13.3 +tomli==2.0.1 +torch==2.0.1 +torchaudio==2.0.2 +torchvision==0.15.2 +tornado==6.3.2 +tqdm==4.65.0 +traitlets==5.9.0 +transformers==4.30.2 +triton==2.0.0 +typing-extensions==4.7.1 +typing-inspect==0.9.0 +tzdata==2023.3 +urllib3==2.0.3 +wandb==0.15.4 +wcwidth==0.2.6 +webencodings==0.5.1 +websocket-client==1.6.1 +werkzeug==2.3.6 +wheel==0.40.0 +xformers==0.0.20 +xxhash==3.2.0 +yarl==1.9.2 +zipp==3.15.0 \ No newline at end of file diff --git a/wandb/run-20230711_182238-06r7b4m3/files/wandb-metadata.json b/wandb/run-20230711_182238-06r7b4m3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..36bb94a3410936c3ea3a79fa87509a14f78727e3 --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/files/wandb-metadata.json @@ -0,0 +1,431 @@ +{ + "os": "Linux-5.15.0-52-generic-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2023-07-11T22:22:40.405929", + "startedAt": "2023-07-11T22:22:38.515893", + "docker": null, + "cuda": null, + "args": [ + "--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5", + "--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/", + "--resolution=512", + "--center_crop", + "--random_flip", + "--train_batch_size=2", + "--gradient_accumulation_steps=4", + "--num_train_epochs=300", + "--learning_rate=1e-5", + "--max_grad_norm=1", + "--lr_scheduler=cosine", + "--lr_warmup_steps=500", + "--output_dir=/l/vision/v5/sragas/easel_ai/models_plain/", + "--checkpointing_steps=500", + "--report_to=wandb", + "--validation_prompt=tom cruise thumbs up", + "--seed=15", + "--push_to_hub", + "--hub_model_id=person-thumbs-up-plain-lora" + ], + "state": "running", + "program": "train_text_to_image_lora.py", + "codePath": "train_text_to_image_lora.py", + "host": "snorlax", + "username": "sragas", + "executable": "/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3", + "cpu_count": 36, + "cpu_count_logical": 72, + "cpu_freq": { + "current": 1158.2238333333332, + "min": 1000.0, + "max": 3900.0 + }, + "cpu_freq_per_core": [ + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 2700.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1100.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 3614.641, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 2000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1100.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1100.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 2500.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 2043.382, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1100.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1100.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 2200.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 2400.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 1000.0, + "min": 1000.0, + "max": 3900.0 + }, + { + "current": 3700.0, + "min": 1000.0, + "max": 3900.0 + } + ], + "disk": { + "total": 490.5871887207031, + "used": 102.94281387329102 + }, + "gpu": "Quadro RTX 6000", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Quadro RTX 6000", + "memory_total": 25769803776 + }, + { + "name": "Quadro RTX 6000", + "memory_total": 25769803776 + }, + { + "name": "Quadro RTX 6000", + "memory_total": 25769803776 + }, + { + "name": "Quadro RTX 6000", + "memory_total": 25769803776 + } + ], + "memory": { + "total": 502.5611343383789 + } +} diff --git a/wandb/run-20230711_182238-06r7b4m3/files/wandb-summary.json b/wandb/run-20230711_182238-06r7b4m3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/files/wandb-summary.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/wandb/run-20230711_182238-06r7b4m3/logs/debug-internal.log b/wandb/run-20230711_182238-06r7b4m3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7b9a4a0721413954dd3f5de3b7ee89abb0d7323b --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/logs/debug-internal.log @@ -0,0 +1,4510 @@ +2023-07-11 18:22:38,542 INFO StreamThr :2302108 [internal.py:wandb_internal():86] W&B internal server running at pid: 2302108, started at: 2023-07-11 18:22:38.540909 +2023-07-11 18:22:38,544 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status +2023-07-11 18:22:38,546 INFO WriterThread:2302108 [datastore.py:open_for_write():85] open: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb +2023-07-11 18:22:38,550 DEBUG SenderThread:2302108 [sender.py:send():369] send: header +2023-07-11 18:22:38,550 DEBUG SenderThread:2302108 [sender.py:send():369] send: run +2023-07-11 18:22:38,752 ERROR SenderThread:2302108 [internal_api.py:execute():323] 500 response executing GraphQL. +2023-07-11 18:22:38,752 ERROR SenderThread:2302108 [internal_api.py:execute():324] {"errors":[{"message":"An internal error occurred. Please contact support.","path":["upsertBucket"]}],"data":{"upsertBucket":null}} +2023-07-11 18:22:40,240 INFO SenderThread:2302108 [dir_watcher.py:__init__():211] watching files in: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files +2023-07-11 18:22:40,240 INFO SenderThread:2302108 [sender.py:_start_run_threads():1100] run started: 06r7b4m3 with start time 1689114158.539519 +2023-07-11 18:22:40,240 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: summary_record +2023-07-11 18:22:40,242 INFO SenderThread:2302108 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-11 18:22:40,246 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: check_version +2023-07-11 18:22:40,246 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: check_version +2023-07-11 18:22:40,333 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: run_start +2023-07-11 18:22:40,368 DEBUG HandlerThread:2302108 [system_info.py:__init__():31] System info init +2023-07-11 18:22:40,368 DEBUG HandlerThread:2302108 [system_info.py:__init__():46] System info init done +2023-07-11 18:22:40,368 INFO HandlerThread:2302108 [system_monitor.py:start():181] Starting system monitor +2023-07-11 18:22:40,368 INFO SystemMonitor:2302108 [system_monitor.py:_start():145] Starting system asset monitoring threads +2023-07-11 18:22:40,369 INFO HandlerThread:2302108 [system_monitor.py:probe():201] Collecting system info +2023-07-11 18:22:40,370 INFO SystemMonitor:2302108 [interfaces.py:start():190] Started cpu monitoring +2023-07-11 18:22:40,371 INFO SystemMonitor:2302108 [interfaces.py:start():190] Started disk monitoring +2023-07-11 18:22:40,372 INFO SystemMonitor:2302108 [interfaces.py:start():190] Started gpu monitoring +2023-07-11 18:22:40,373 INFO SystemMonitor:2302108 [interfaces.py:start():190] Started memory monitoring +2023-07-11 18:22:40,375 INFO SystemMonitor:2302108 [interfaces.py:start():190] Started network monitoring +2023-07-11 18:22:40,405 DEBUG HandlerThread:2302108 [system_info.py:probe():195] Probing system +2023-07-11 18:22:40,416 DEBUG HandlerThread:2302108 [gitlib.py:_init_repo():53] git repository is invalid +2023-07-11 18:22:40,416 DEBUG HandlerThread:2302108 [system_info.py:probe():240] Probing system done +2023-07-11 18:22:40,416 DEBUG HandlerThread:2302108 [system_monitor.py:probe():210] {'os': 'Linux-5.15.0-52-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2023-07-11T22:22:40.405929', 'startedAt': '2023-07-11T22:22:38.515893', 'docker': None, 'cuda': None, 'args': ('--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5', '--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/', '--resolution=512', '--center_crop', '--random_flip', '--train_batch_size=2', '--gradient_accumulation_steps=4', '--num_train_epochs=300', '--learning_rate=1e-5', '--max_grad_norm=1', '--lr_scheduler=cosine', '--lr_warmup_steps=500', '--output_dir=/l/vision/v5/sragas/easel_ai/models_plain/', '--checkpointing_steps=500', '--report_to=wandb', '--validation_prompt=tom cruise thumbs up', '--seed=15', '--push_to_hub', '--hub_model_id=person-thumbs-up-plain-lora'), 'state': 'running', 'program': 'train_text_to_image_lora.py', 'codePath': 'train_text_to_image_lora.py', 'host': 'snorlax', 'username': 'sragas', 'executable': '/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3', 'cpu_count': 36, 'cpu_count_logical': 72, 'cpu_freq': {'current': 1158.2238333333332, 'min': 1000.0, 'max': 3900.0}, 'cpu_freq_per_core': [{'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 2700.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1100.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 3614.641, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 2000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1100.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1100.0, 'min': 1000.0, 'max': 3900.0}, {'current': 2500.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 2043.382, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1100.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1100.0, 'min': 1000.0, 'max': 3900.0}, {'current': 2200.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 2400.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 1000.0, 'min': 1000.0, 'max': 3900.0}, {'current': 3700.0, 'min': 1000.0, 'max': 3900.0}], 'disk': {'total': 490.5871887207031, 'used': 102.94281387329102}, 'gpu': 'Quadro RTX 6000', 'gpu_count': 4, 'gpu_devices': [{'name': 'Quadro RTX 6000', 'memory_total': 25769803776}, {'name': 'Quadro RTX 6000', 'memory_total': 25769803776}, {'name': 'Quadro RTX 6000', 'memory_total': 25769803776}, {'name': 'Quadro RTX 6000', 'memory_total': 25769803776}], 'memory': {'total': 502.5611343383789}} +2023-07-11 18:22:40,416 INFO HandlerThread:2302108 [system_monitor.py:probe():211] Finished collecting system info +2023-07-11 18:22:40,416 INFO HandlerThread:2302108 [system_monitor.py:probe():214] Publishing system info +2023-07-11 18:22:40,416 DEBUG HandlerThread:2302108 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2023-07-11 18:22:40,419 DEBUG HandlerThread:2302108 [system_info.py:_save_pip():67] Saving pip packages done +2023-07-11 18:22:40,446 INFO HandlerThread:2302108 [system_monitor.py:probe():216] Finished publishing system info +2023-07-11 18:22:40,455 DEBUG SenderThread:2302108 [sender.py:send():369] send: files +2023-07-11 18:22:40,456 INFO SenderThread:2302108 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now +2023-07-11 18:22:40,470 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:22:40,471 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:22:40,544 DEBUG SenderThread:2302108 [sender.py:send():369] send: telemetry +2023-07-11 18:22:40,799 INFO wandb-upload_0:2302108 [upload_job.py:push():133] Uploaded file /tmp/tmpeie9f67kwandb/ef3hp22k-wandb-metadata.json +2023-07-11 18:22:41,245 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/requirements.txt +2023-07-11 18:22:41,246 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/wandb-metadata.json +2023-07-11 18:22:41,246 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/output.log +2023-07-11 18:22:41,246 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/wandb-summary.json +2023-07-11 18:22:43,244 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/output.log +2023-07-11 18:22:44,551 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:22:47,249 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/output.log +2023-07-11 18:22:50,417 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:22:55,418 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:22:55,562 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:22:55,563 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:23:00,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:05,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:10,562 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:23:10,562 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:23:11,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:12,272 INFO Thread-12 :2302108 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/files/config.yaml +2023-07-11 18:23:16,792 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:21,793 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:25,562 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:23:25,563 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:23:27,606 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:32,607 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:37,608 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:40,375 DEBUG SystemMonitor:2302108 [system_monitor.py:_start():159] Starting system metrics aggregation loop +2023-07-11 18:23:40,379 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:23:40,562 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:23:40,563 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:23:43,606 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:48,607 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:53,609 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:23:55,562 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:23:55,564 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:23:59,607 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:04,609 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:09,610 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:10,383 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:24:10,563 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:24:10,563 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:24:14,616 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:19,617 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:24,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:25,572 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:24:25,573 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:24:29,634 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:34,635 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:39,636 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:40,386 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:24:40,573 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:24:40,574 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:24:45,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:50,619 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:24:55,572 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:24:55,573 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:24:55,620 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:00,621 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:05,622 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:10,389 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:25:10,590 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:25:10,591 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:25:10,670 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:15,672 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:20,673 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:25,589 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:25:25,590 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:25:25,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:30,722 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:35,723 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:40,393 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:25:40,591 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:25:40,591 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:25:41,640 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:46,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:51,642 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:25:55,591 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:25:55,592 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:25:56,653 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:01,654 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:06,655 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:10,395 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:26:10,592 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:26:10,593 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:26:11,679 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:16,680 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:21,681 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:25,650 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:26:25,651 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:26:26,713 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:31,714 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:36,715 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:40,399 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:26:40,651 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:26:40,651 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:26:42,686 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:47,688 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:52,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:26:55,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:26:55,653 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:26:57,690 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:02,691 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:07,692 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:10,402 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:27:10,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:27:10,653 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:27:12,698 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:17,699 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:22,700 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:25,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:27:25,652 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:27:27,733 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:32,735 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:37,737 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:40,407 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:27:40,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:27:40,653 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:27:43,691 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:48,692 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:53,693 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:27:55,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:27:55,653 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:27:58,698 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:03,699 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:08,700 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:10,408 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:28:10,907 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:28:10,907 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:28:13,964 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:18,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:23,966 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:25,906 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:28:25,907 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:28:29,943 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:34,944 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:39,945 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:40,411 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:28:41,617 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:28:41,618 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:28:45,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:50,658 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:55,659 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:28:56,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:28:56,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:29:01,659 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:06,661 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:10,414 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:29:11,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:29:11,618 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:29:11,703 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:16,704 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:21,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:26,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:29:26,618 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:29:27,669 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:32,670 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:37,671 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:40,417 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:29:41,619 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:29:41,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:29:43,667 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:48,668 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:53,669 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:29:56,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:29:56,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:29:59,666 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:04,667 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:09,668 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:10,420 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:30:11,802 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:30:11,803 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:30:14,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:19,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:24,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:26,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:30:26,804 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:30:30,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:35,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:40,424 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:30:41,426 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:41,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:30:41,804 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:30:46,840 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:51,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:30:56,805 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:30:56,805 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:30:57,115 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:02,116 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:07,117 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:10,427 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:31:11,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:31:11,804 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:31:12,843 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:17,844 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:22,846 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:26,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:31:26,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:31:27,868 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:32,870 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:37,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:40,430 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:31:41,807 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:31:41,808 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:31:43,862 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:48,863 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:53,864 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:31:56,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:31:56,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:31:59,860 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:04,862 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:09,863 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:10,434 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:32:11,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:32:11,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:32:15,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:20,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:25,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:26,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:32:26,808 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:32:30,868 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:35,869 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:40,437 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:32:41,439 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:41,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:32:41,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:32:46,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:51,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:32:56,809 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:32:56,811 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:32:57,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:02,853 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:07,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:10,440 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:33:11,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:33:11,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:33:13,850 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:18,851 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:23,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:26,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:33:26,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:33:28,882 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:33,883 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:38,884 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:40,443 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:33:41,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:33:41,809 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:33:44,853 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:49,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:54,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:33:56,809 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:33:56,810 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:34:00,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:05,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:10,446 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:34:11,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:11,810 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:34:11,810 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:34:16,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:21,860 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:26,861 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:27,172 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:34:27,172 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:34:32,212 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:37,214 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:40,449 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:34:42,170 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:34:42,171 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:34:42,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:47,217 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:52,218 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:57,219 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:34:57,524 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:34:57,525 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:35:02,607 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:07,609 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:10,452 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:35:12,520 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:35:12,521 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:35:13,564 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:18,565 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:23,567 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:27,521 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:35:27,522 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:35:28,587 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:33,588 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:38,589 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:40,453 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:35:42,521 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:35:42,521 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:35:44,568 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:49,569 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:54,570 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:35:57,523 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:35:57,524 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:36:00,568 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:05,569 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:10,457 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:36:11,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:13,187 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:36:13,188 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:36:17,231 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:22,232 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:27,233 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:28,186 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:36:28,187 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:36:33,233 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:38,234 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:40,460 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:36:43,462 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:43,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:36:43,722 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:36:48,777 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:53,778 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:36:58,720 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:36:58,722 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:36:59,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:04,772 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:09,773 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:10,463 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:37:13,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:37:13,722 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:37:15,117 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:20,118 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:25,119 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:28,720 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:37:28,721 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:37:30,765 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:35,767 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:40,467 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:37:41,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:43,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:37:43,721 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:37:46,778 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:51,780 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:56,781 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:37:59,030 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:37:59,031 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:38:02,094 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:07,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:10,469 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:38:12,471 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:14,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:38:14,034 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:38:18,088 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:23,089 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:28,090 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:29,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:38:29,034 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:38:34,080 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:39,081 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:40,472 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:38:44,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:38:44,033 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:38:44,091 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:49,093 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:54,094 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:38:59,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:38:59,034 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:39:00,081 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:05,082 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:10,083 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:10,475 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:39:14,080 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:39:14,080 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:39:15,123 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:20,125 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:25,126 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:29,079 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:39:29,080 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:39:30,152 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:35,154 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:40,155 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:40,479 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:39:44,079 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:39:44,080 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:39:45,190 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:50,191 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:55,192 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:39:59,079 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:39:59,081 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:40:01,123 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:06,124 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:10,482 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:40:11,484 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:14,080 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:40:14,081 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:40:17,132 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:22,133 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:27,134 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:29,265 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:40:29,265 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:40:32,317 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:37,318 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:40,484 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:40:42,487 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:44,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:40:44,300 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:40:48,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:53,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:58,357 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:40:59,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:40:59,300 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:41:04,346 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:09,347 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:10,487 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:41:14,490 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:14,617 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:41:14,618 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:41:19,673 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:24,674 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:29,619 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:41:29,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:41:30,661 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:35,662 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:40,491 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:41:41,493 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:44,617 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:41:44,618 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:41:46,681 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:51,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:56,684 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:41:59,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:41:59,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:42:02,660 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:07,661 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:10,495 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:42:13,497 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:14,619 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:42:14,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:42:18,684 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:23,685 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:28,686 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:29,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:42:29,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:42:34,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:39,658 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:40,496 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:42:44,619 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:42:44,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:42:44,673 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:49,674 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:54,675 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:42:59,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:42:59,619 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:42:59,698 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:04,700 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:09,701 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:10,499 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:43:15,467 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:15,468 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:43:15,468 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:43:20,529 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:25,530 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:30,466 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:43:30,467 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:43:31,513 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:36,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:40,501 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:43:42,503 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:45,466 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:43:45,467 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:43:47,582 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:52,583 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:43:57,584 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:00,466 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:44:00,467 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:44:03,523 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:08,524 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:10,504 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:44:14,506 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:15,559 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:44:15,560 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:44:19,609 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:24,611 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:29,612 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:30,560 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:44:30,561 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:44:35,606 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:40,507 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:44:41,509 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:45,565 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:44:45,565 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:44:46,620 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:51,621 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:44:56,623 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:00,564 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:45:00,565 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:45:02,613 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:07,615 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:10,510 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:45:13,513 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:15,566 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:45:15,567 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:45:18,640 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:23,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:28,643 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:30,564 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:45:30,565 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:45:34,629 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:39,630 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:40,513 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:45:45,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:45,565 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:45:45,565 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:45:50,637 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:45:55,638 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:00,568 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:46:00,569 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:46:01,628 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:06,629 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:10,516 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:46:12,518 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:15,570 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:46:15,570 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:46:17,624 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:22,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:27,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:30,569 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:46:30,570 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:46:33,617 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:38,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:40,519 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:46:44,522 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:46,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:46:46,236 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:46:50,303 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:46:55,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:00,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:01,235 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:47:01,236 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:47:06,278 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:10,525 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:47:11,527 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:16,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:47:16,401 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:47:17,450 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:22,451 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:27,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:31,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:47:31,400 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:47:32,462 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:37,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:40,525 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:47:42,528 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:46,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:47:46,400 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:47:48,439 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:53,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:47:58,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:01,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:48:01,401 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:48:04,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:09,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:10,528 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:48:14,530 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:16,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:48:16,401 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:48:20,468 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:25,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:30,471 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:31,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:48:31,401 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:48:35,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:40,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:40,531 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:48:45,533 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:46,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:48:46,401 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:48:51,455 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:48:56,456 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:01,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:49:01,402 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:49:02,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:07,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:10,534 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:49:12,537 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:16,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:49:16,855 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:49:17,930 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:22,931 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:27,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:31,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:49:31,855 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:49:33,920 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:38,921 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:40,537 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:49:44,540 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:46,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:49:46,855 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:49:49,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:54,896 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:49:59,898 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:01,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:50:01,856 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:50:04,898 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:09,899 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:10,540 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:50:15,542 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:16,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:50:16,856 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:50:20,944 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:25,945 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:30,946 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:31,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:50:31,857 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:50:36,904 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:40,543 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:50:42,545 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:46,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:50:46,856 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:50:47,897 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:52,899 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:50:57,900 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:01,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:51:01,857 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:51:02,905 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:07,907 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:10,546 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:51:13,548 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:16,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:51:16,857 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:51:18,905 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:23,906 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:28,907 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:31,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:51:31,966 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:51:34,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:39,014 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:40,549 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:51:44,552 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:47,116 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:51:47,118 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:51:50,207 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:51:55,208 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:00,209 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:02,116 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:52:02,116 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:52:06,169 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:10,553 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:52:11,555 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:16,557 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:17,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:52:17,443 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:52:22,506 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:27,508 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:32,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:52:32,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:52:33,486 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:38,487 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:40,555 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:52:43,558 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:47,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:52:47,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:52:49,522 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:54,523 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:52:59,524 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:02,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:53:02,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:53:05,479 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:10,480 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:10,558 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:53:15,561 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:17,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:53:17,442 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:53:21,499 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:26,501 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:31,502 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:32,449 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:53:32,449 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:53:36,510 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:40,560 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:53:41,562 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:46,563 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:47,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:53:47,449 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:53:52,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:53:57,517 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:02,518 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:02,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:54:02,859 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:54:07,899 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:10,563 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:54:13,565 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:17,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:54:17,858 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:54:18,906 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:23,907 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:28,909 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:32,860 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:54:32,860 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:54:34,903 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:39,904 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:40,566 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:54:45,569 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:47,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:54:47,858 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:54:50,914 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:54:55,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:00,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:02,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:55:02,858 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:55:06,905 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:10,569 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:55:12,571 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:17,572 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:17,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:55:17,859 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:55:22,909 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:27,910 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:32,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:55:32,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:55:32,922 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:37,923 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:40,572 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:55:43,574 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:47,879 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:55:47,880 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:55:48,919 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:53,921 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:55:58,922 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:02,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:56:02,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:56:04,920 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:09,922 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:10,575 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:56:15,578 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:17,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:56:17,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:56:20,927 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:25,928 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:30,929 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:32,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:56:32,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:56:36,918 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:40,577 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:56:42,579 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:47,580 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:47,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:56:47,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:56:52,951 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:56:57,952 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:02,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:57:02,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:57:03,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:08,933 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:10,581 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:57:14,584 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:17,881 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:57:17,881 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:57:19,927 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:24,929 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:29,930 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:32,882 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:57:32,883 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:57:34,931 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:39,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:40,582 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:57:45,585 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:47,881 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:57:47,882 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:57:50,931 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:57:55,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:00,933 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:02,881 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:58:02,882 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:58:06,923 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:10,585 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:58:12,587 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:17,588 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:17,881 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:58:17,882 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:58:22,926 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:27,927 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:32,882 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:58:32,883 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:58:32,931 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:37,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:40,588 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:58:43,590 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:47,882 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:58:47,883 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:58:48,923 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:53,925 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:58:58,926 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:03,098 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:59:03,099 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:59:04,148 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:09,149 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:10,592 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:59:14,594 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:18,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:59:18,097 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:59:20,143 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:25,144 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:30,145 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:33,098 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:59:33,099 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:59:35,157 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:40,158 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:40,594 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 18:59:45,597 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:48,097 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 18:59:48,097 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 18:59:51,215 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 18:59:56,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:01,217 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:03,097 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:00:03,097 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:00:07,147 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:10,597 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:00:12,599 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:17,601 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:18,097 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:00:18,098 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:00:23,155 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:28,156 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:33,157 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:33,489 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:00:33,490 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:00:38,538 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:40,600 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:00:43,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:48,489 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:00:48,490 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:00:49,554 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:54,555 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:00:59,557 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:03,489 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:01:03,490 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:01:05,545 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:10,546 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:10,603 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:01:15,606 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:18,489 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:01:18,490 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:01:21,530 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:26,531 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:31,532 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:33,827 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:01:33,828 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:01:36,868 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:40,608 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:01:42,610 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:47,611 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:49,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:01:49,435 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:01:53,490 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:01:58,491 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:03,492 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:04,933 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:02:04,934 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:02:08,981 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:10,609 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:02:14,612 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:19,613 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:19,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:02:19,933 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:02:24,982 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:29,983 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:34,933 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:02:34,934 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:02:35,010 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:40,011 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:40,612 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:02:45,615 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:50,234 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:02:50,235 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:02:51,300 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:02:56,301 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:01,302 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:05,233 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:03:05,234 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:03:07,290 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:10,615 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:03:12,617 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:17,618 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:20,396 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:03:20,396 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:03:23,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:28,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:33,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:35,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:03:35,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:03:38,573 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:40,618 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:03:43,621 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:48,622 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:50,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:03:50,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:03:54,496 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:03:59,497 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:04,498 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:05,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:04:05,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:04:10,483 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:10,621 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:04:15,624 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:20,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:04:20,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:04:21,481 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:26,482 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:31,484 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:35,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:04:35,442 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:04:36,567 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:40,624 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:04:41,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:46,627 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:50,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:04:50,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:04:52,480 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:04:57,481 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:02,482 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:05,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:05:05,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:05:07,492 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:10,627 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:05:12,629 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:17,631 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:20,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:05:20,441 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:05:23,478 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:28,480 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:33,481 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:35,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:05:35,442 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:05:38,514 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:40,630 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:05:43,632 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:48,634 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:50,812 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:05:50,814 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:05:53,869 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:05:58,870 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:03,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:06,060 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:06:06,061 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:06:09,100 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:10,633 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:06:14,635 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:19,636 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:21,059 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:06:21,060 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:06:25,100 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:30,101 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:35,102 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:36,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:06:36,460 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:06:40,505 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:40,637 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:06:45,639 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:50,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:06:51,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:06:51,460 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:06:56,499 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:01,500 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:06,501 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:06,536 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:07:06,536 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:07:10,639 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:07:11,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:16,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:21,535 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:07:21,536 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:07:22,580 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:27,581 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:32,582 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:36,549 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:07:36,550 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:07:37,597 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:40,642 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:07:42,644 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:47,645 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:51,549 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:07:51,550 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:07:53,587 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:07:58,588 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:03,589 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:06,550 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:08:06,551 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:08:09,588 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:10,645 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:08:14,647 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:19,649 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:22,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:08:22,114 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:08:25,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:30,161 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:35,162 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:37,113 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:08:37,114 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:08:40,648 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:08:40,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:45,651 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:50,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:08:52,559 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:08:52,560 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:08:56,605 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:01,606 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:06,607 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:07,558 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:09:07,559 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:09:10,651 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:09:11,653 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:16,653 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:21,655 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:22,558 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:09:22,559 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:09:27,604 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:32,605 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:37,579 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:09:37,580 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:09:37,615 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:40,654 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:09:42,656 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:47,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:52,580 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:09:52,581 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:09:53,619 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:09:58,620 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:03,621 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:07,578 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:10:07,579 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:10:08,623 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:10,656 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:10:13,659 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:18,660 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:22,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:10:22,603 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:10:24,648 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:29,649 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:34,650 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:37,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:10:37,603 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:10:40,648 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:40,659 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:10:45,662 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:50,663 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:10:52,801 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:10:52,802 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:10:55,844 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:00,845 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:05,846 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:07,801 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:11:07,802 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:11:10,663 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:11:11,665 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:16,666 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:21,667 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:22,809 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:11:22,810 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:11:26,851 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:31,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:36,853 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:38,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:11:38,689 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:11:40,667 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:11:42,669 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:47,670 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:52,671 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:11:53,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:11:53,690 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:11:57,737 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:02,738 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:07,739 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:08,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:12:08,690 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:12:10,669 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:12:13,671 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:18,672 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:23,673 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:23,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:12:23,689 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:12:28,726 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:33,728 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:38,729 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:38,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:12:38,841 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:12:40,672 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:12:44,674 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:49,675 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:53,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:12:53,842 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:12:54,882 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:12:59,883 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:04,884 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:08,840 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:13:08,841 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:13:09,899 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:10,675 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:13:15,677 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:20,678 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:24,195 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:13:24,196 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:13:26,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:31,238 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:36,239 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:39,196 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:13:39,197 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:13:40,677 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:13:41,679 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:46,681 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:51,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:13:54,214 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:13:54,215 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:13:57,295 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:02,296 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:07,297 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:09,247 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:14:09,248 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:14:10,681 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:14:12,683 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:17,684 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:22,685 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:24,246 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:14:24,247 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:14:28,297 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:33,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:38,300 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:39,247 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:14:39,248 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:14:40,683 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:14:43,685 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:48,686 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:53,687 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:14:54,247 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:14:54,248 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:14:59,284 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:04,285 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:09,248 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:15:09,249 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:15:09,308 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:10,686 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:15:14,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:19,690 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:24,262 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:15:24,263 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:15:25,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:30,307 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:35,308 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:39,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:15:39,344 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:15:40,391 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:40,689 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:15:45,692 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:50,693 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:15:54,344 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:15:54,345 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:15:56,406 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:01,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:06,408 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:09,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:16:09,344 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:16:10,692 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:16:11,694 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:16,695 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:21,696 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:24,344 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:16:24,345 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:16:27,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:32,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:37,395 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:39,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:16:39,344 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:16:40,696 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:16:42,698 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:47,699 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:52,700 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:16:54,472 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:16:54,473 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:16:58,547 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:03,548 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:08,550 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:09,472 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:17:09,473 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:17:10,698 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:17:13,700 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:18,701 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:23,702 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:24,472 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:17:24,473 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:17:29,552 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:34,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:39,473 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:17:39,474 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:17:40,524 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:40,701 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:17:45,703 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:50,704 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:17:54,473 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:17:54,473 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:17:56,529 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:01,530 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:06,531 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:09,473 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:18:09,474 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:18:10,704 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:18:11,706 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:16,707 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:21,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:24,474 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:18:24,475 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:18:27,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:32,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:37,517 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:39,685 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:18:39,686 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:18:40,707 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:18:42,709 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:47,710 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:52,711 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:18:55,533 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:18:55,534 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:18:58,608 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:03,609 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:08,611 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:10,532 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:19:10,533 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:19:10,709 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:19:13,712 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:18,713 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:23,714 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:25,533 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:19:25,533 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:19:29,595 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:34,596 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:39,597 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:40,533 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:19:40,533 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:19:40,712 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:19:44,715 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:49,716 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:54,717 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:19:55,534 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:19:55,534 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:20:00,595 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:05,596 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:10,586 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:20:10,587 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:20:10,665 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:10,715 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:20:15,718 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:20,720 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:25,587 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:20:25,588 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:20:26,639 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:31,640 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:36,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:40,719 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:20:40,992 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:20:40,993 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:20:42,045 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:47,046 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:52,047 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:20:55,992 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:20:55,993 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:20:58,030 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:03,031 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:08,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:10,723 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:21:11,011 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:21:11,012 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:21:13,060 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:18,061 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:23,062 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:26,011 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:21:26,012 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:21:29,058 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:34,059 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:39,060 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:40,724 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:21:41,013 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:21:41,014 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:21:45,051 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:50,052 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:55,053 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:21:56,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:21:56,013 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:22:00,058 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:05,059 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:10,060 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:10,727 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:22:11,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:22:11,012 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:22:16,052 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:21,053 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:26,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:22:26,013 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:22:26,064 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:31,066 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:36,067 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:40,730 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:22:41,211 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:22:41,211 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:22:41,279 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:46,280 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:51,281 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:56,282 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:22:56,751 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:22:56,751 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:23:01,884 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:06,885 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:10,733 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:23:11,749 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:23:11,749 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:23:12,831 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:17,832 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:22,833 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:26,750 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:23:26,750 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:23:28,831 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:33,832 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:38,833 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:40,736 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:23:41,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:23:41,756 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:23:44,791 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:49,792 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:54,793 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:23:56,754 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:23:56,755 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:24:01,756 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:24:06,757 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:24:08,064 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:10,739 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:24:11,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:24:11,755 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:24:13,809 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:18,811 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:23,812 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:26,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:24:26,756 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:24:26,756 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:24:28,835 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:33,836 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:38,837 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:40,742 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:24:41,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:24:41,814 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:24:41,814 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:24:43,887 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:48,888 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:53,889 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:24:56,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:24:56,814 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:24:58,938 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:03,940 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:08,941 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:10,745 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:25:11,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:25:11,814 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:25:14,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:19,855 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:24,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:26,814 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:25:26,815 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:25:26,815 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:25:29,911 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:34,912 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:39,914 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:40,749 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:25:42,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:25:42,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:25:42,387 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:25:45,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:50,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:55,446 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:25:57,766 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:25:57,766 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:25:57,767 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:26:00,821 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:05,822 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:10,753 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:26:11,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:12,819 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:26:12,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:26:12,820 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:26:16,875 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:21,876 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:26,877 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:27,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:26:27,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:26:27,821 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:26:32,875 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:37,876 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:40,754 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:26:42,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:26:42,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:26:42,821 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:26:42,881 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:47,882 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:52,883 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:26:57,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:26:57,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:26:57,820 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:26:58,869 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:03,873 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:08,874 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:10,757 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:27:12,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:27:12,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:27:12,821 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:27:13,888 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:18,889 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:23,890 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:28,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:27:28,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:27:28,454 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:27:29,498 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:34,499 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:39,500 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:40,760 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:27:43,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:27:43,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:27:43,453 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:27:44,534 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:49,535 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:54,536 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:27:58,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:27:58,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:27:58,453 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:28:00,492 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:05,493 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:10,495 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:10,763 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:28:13,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:28:13,496 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:28:13,497 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:28:15,547 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:20,548 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:25,549 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:28,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:28:28,454 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:28:28,454 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:28:31,511 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:36,512 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:40,766 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:28:41,768 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:43,599 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:28:43,599 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:28:43,600 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:28:47,649 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:52,651 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:57,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:28:58,599 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:28:58,600 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:28:58,600 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:29:03,644 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:08,645 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:10,769 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:29:13,599 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:29:13,600 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:29:13,600 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:29:13,679 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:18,680 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:23,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:28,600 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:29:28,601 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:29:28,601 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:29:29,666 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:34,667 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:39,669 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:40,772 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:29:43,600 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:29:43,600 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:29:43,601 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:29:45,644 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:50,645 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:55,646 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:29:58,789 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:29:58,789 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:29:58,790 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:30:00,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:05,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:10,775 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:30:11,777 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:13,901 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:30:13,901 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:30:13,902 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:30:16,950 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:21,951 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:26,952 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:29,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:30:29,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:30:29,772 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:30:32,822 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:37,823 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:40,778 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:30:43,780 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:44,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:30:44,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:30:44,772 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:30:48,812 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:53,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:58,814 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:30:59,770 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:30:59,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:30:59,771 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:31:03,822 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:08,824 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:10,782 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:31:14,785 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:14,817 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:31:14,817 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:31:14,818 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:31:19,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:24,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:29,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:31:29,859 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:29,860 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:31:29,861 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:31:34,914 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:39,915 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:40,784 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:31:44,817 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:31:44,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:31:44,818 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:31:45,862 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:50,863 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:55,864 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:31:59,817 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:31:59,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:31:59,818 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:32:00,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:05,872 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:10,787 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:32:11,789 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:14,817 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:32:14,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:32:14,818 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:32:16,872 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:21,873 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:26,875 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:29,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:32:29,819 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:32:29,819 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:32:32,863 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:37,864 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:40,790 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:32:43,792 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:44,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:32:44,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:32:44,819 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:32:48,869 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:53,870 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:32:58,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:01,140 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:33:01,140 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:33:01,141 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:33:04,187 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:09,189 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:10,793 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:33:14,795 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:16,140 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:33:16,140 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:33:16,141 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:33:20,218 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:25,219 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:30,220 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:31,141 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:33:31,141 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:33:31,142 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:33:36,202 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:40,796 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:33:41,798 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:46,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:33:46,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:33:46,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:33:47,320 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:52,321 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:33:57,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:01,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:34:01,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:34:01,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:34:02,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:07,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:10,798 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:34:12,801 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:16,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:34:16,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:34:16,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:34:18,312 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:23,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:28,314 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:31,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:34:31,316 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:34:31,317 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:34:33,370 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:38,371 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:40,801 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:34:43,804 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:46,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:34:46,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:34:49,346 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:54,347 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:34:59,349 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:01,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:35:01,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:35:01,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:35:05,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:10,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:10,804 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:35:15,807 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:16,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:35:16,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:35:16,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:35:21,358 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:26,359 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:31,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:35:31,276 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:35:31,277 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:35:32,333 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:37,335 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:40,809 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:35:42,812 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:46,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:35:46,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:35:48,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:53,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:35:58,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:01,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:36:01,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:36:04,316 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:09,318 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:10,810 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:36:14,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:16,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:36:16,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:36:16,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:36:20,317 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:25,319 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:30,320 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:31,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:36:31,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:36:35,333 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:40,335 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:40,813 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:36:45,816 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:46,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:36:46,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:36:51,335 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:36:56,337 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:01,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:37:01,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:37:01,299 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:37:01,371 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:06,373 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:10,816 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:37:11,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:16,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:37:16,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:37:16,299 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:37:17,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:22,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:27,344 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:31,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:37:31,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:37:31,344 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:37:32,388 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:37,389 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:40,819 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:37:42,821 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:46,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:37:46,343 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:37:48,390 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:53,391 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:37:58,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:01,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:38:01,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:38:01,410 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:38:03,458 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:08,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:10,822 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:38:13,824 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:16,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:38:16,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:38:16,411 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:38:19,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:24,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:29,454 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:31,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:38:31,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:38:31,411 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:38:34,465 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:39,467 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:40,826 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:38:44,828 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:46,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:38:46,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:38:46,411 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:38:50,462 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:38:55,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:00,464 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:01,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:39:01,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:39:01,411 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:39:06,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:10,829 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:39:11,831 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:16,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:39:16,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:39:16,453 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:39:17,504 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:22,506 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:27,507 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:31,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:39:31,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:39:31,411 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:39:33,473 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:38,474 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:40,831 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:39:43,834 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:46,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:39:46,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:39:46,412 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:39:49,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:54,465 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:39:59,466 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:01,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:40:01,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:40:01,962 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:40:05,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:10,007 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:10,834 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:40:15,837 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:16,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:40:16,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:40:16,962 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:40:21,016 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:26,018 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:31,019 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:31,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:40:31,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:40:31,963 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:40:36,020 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:40,839 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:40:41,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:46,842 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:46,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:40:46,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:40:46,962 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:40:52,010 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:40:57,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:01,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:41:01,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:41:01,962 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:41:03,008 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:08,009 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:10,840 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:41:13,842 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:17,148 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:41:17,148 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:41:17,149 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:41:19,197 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:24,198 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:29,199 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:32,149 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:41:32,149 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:41:32,149 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:41:34,211 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:39,212 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:40,842 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:41:44,845 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:47,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:41:47,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:41:47,341 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:41:50,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:41:55,387 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:00,388 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:02,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:42:02,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:42:02,341 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:42:06,387 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:10,845 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:42:11,848 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:16,849 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:17,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:42:17,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:42:17,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:42:22,389 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:27,390 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:32,391 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:32,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:42:32,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:42:32,444 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:42:37,577 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:40,849 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:42:42,851 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:47,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:42:47,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:42:47,444 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:42:48,501 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:53,503 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:42:58,504 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:02,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:43:02,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:43:02,444 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:43:04,490 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:09,491 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:10,851 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:43:14,854 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:17,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:43:17,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:43:17,444 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:43:20,514 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:25,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:30,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:32,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:43:32,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:43:32,445 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:43:36,507 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:40,854 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:43:41,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:46,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:47,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:43:47,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:43:47,445 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:43:52,496 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:43:57,497 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:02,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:44:02,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:44:02,445 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:44:03,493 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:08,494 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:10,857 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:44:13,860 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:17,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:44:17,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:44:17,804 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:44:19,849 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:24,850 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:29,851 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:32,804 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:44:32,804 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:44:32,805 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:44:34,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:39,857 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:40,860 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:44:44,863 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:48,312 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:44:48,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:44:48,313 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:44:50,372 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:44:55,373 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:00,375 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:03,312 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:45:03,312 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:45:03,313 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:45:06,351 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:10,863 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:45:11,865 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:16,866 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:18,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:45:18,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:45:18,314 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:45:22,352 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:27,353 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:32,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:33,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:45:33,356 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:45:33,357 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:45:37,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:40,867 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:45:42,869 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:47,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:48,312 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:45:48,313 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:45:53,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:45:58,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:03,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:46:03,314 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:46:03,359 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:08,361 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:10,869 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:46:13,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:18,314 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:46:18,314 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:46:18,314 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:46:19,352 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:24,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:29,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:33,314 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:46:33,314 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:46:33,315 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:46:35,351 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:40,352 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:40,872 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:46:45,875 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:48,364 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:46:48,404 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:46:48,405 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:46:51,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:46:56,464 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:01,465 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:03,364 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:47:03,365 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:47:07,422 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:10,875 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:47:12,877 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:17,878 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:18,367 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:47:18,367 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:47:18,367 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:47:23,464 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:28,465 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:33,367 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:47:33,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:47:33,368 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:47:34,413 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:39,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:40,877 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:47:44,880 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:48,389 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:47:48,390 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:47:48,390 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:47:50,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:47:55,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:00,455 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:03,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:48:03,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:48:03,394 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:48:05,574 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:10,575 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:10,881 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:48:15,884 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:18,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:48:18,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:48:18,394 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:48:21,431 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:26,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:31,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:33,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:48:33,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:48:33,437 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:48:36,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:40,884 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:48:41,886 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:46,887 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:48,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:48:48,393 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:48:52,435 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:48:57,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:02,438 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:03,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:49:03,393 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:49:07,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:10,887 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:49:12,889 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:17,890 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:18,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:49:18,394 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:49:23,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:28,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:33,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:49:33,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:49:33,395 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:49:33,460 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:38,461 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:40,890 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:49:43,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:48,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:49:48,394 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:49:49,432 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:54,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:49:59,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:03,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:50:03,394 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:50:04,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:09,447 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:10,893 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:50:14,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:18,397 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:50:18,398 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:50:18,398 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:50:20,454 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:25,455 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:30,456 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:33,397 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:50:33,398 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:50:36,449 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:40,914 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:50:41,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:46,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:48,398 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:50:48,398 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:50:48,399 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:50:52,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:50:57,446 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:02,447 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:03,398 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:51:03,398 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:51:08,439 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:10,898 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:51:13,900 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:18,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:51:18,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:51:18,400 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:51:19,449 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:24,451 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:29,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:33,577 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:51:33,577 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:51:33,578 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:51:34,633 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:39,635 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:40,905 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:51:44,907 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:48,577 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:51:48,577 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:51:50,630 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:51:55,631 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:00,632 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:03,979 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:52:03,980 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:52:03,980 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:52:06,049 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:10,904 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:52:11,906 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:16,908 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:19,197 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:52:19,198 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:52:19,198 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:52:22,260 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:27,261 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:32,262 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:34,197 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:52:34,198 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:52:37,281 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:40,907 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:52:42,910 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:47,911 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:49,699 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:52:49,699 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:52:49,700 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:52:53,770 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:52:58,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:03,772 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:04,698 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:53:04,699 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:53:09,739 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:10,910 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:53:14,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:19,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:53:19,819 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:53:19,819 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:53:20,871 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:25,872 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:30,873 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:35,297 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:53:35,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:53:35,298 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:53:36,348 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:40,913 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:53:41,915 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:46,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:50,297 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:53:50,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:53:50,298 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:53:52,353 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:53:57,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:02,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:05,297 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:54:05,298 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:54:08,339 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:10,916 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:54:13,918 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:18,919 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:20,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:54:20,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:54:20,299 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:54:24,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:29,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:34,364 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:35,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:54:35,299 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:54:40,336 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:40,919 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:54:45,922 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:50,298 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:54:50,299 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:54:51,345 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:54:56,346 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:01,347 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:05,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:55:05,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:55:05,300 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:55:07,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:10,923 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:55:13,925 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:18,927 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:20,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:55:20,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:55:20,603 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:55:24,662 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:29,664 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:34,665 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:35,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:55:35,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:55:35,603 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:55:40,651 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:40,925 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:55:45,928 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:50,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:55:50,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:55:50,603 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:55:51,655 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:55:56,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:01,658 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:05,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:56:05,602 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:56:05,603 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:56:07,640 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:10,928 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:56:12,930 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:17,931 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:20,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:56:20,856 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:56:20,857 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:56:23,912 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:28,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:33,914 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:36,332 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:56:36,332 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:56:36,333 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:56:39,378 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:40,931 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:56:44,933 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:49,934 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:56:51,332 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:56:51,332 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:56:51,333 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:56:55,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:00,385 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:05,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:06,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:57:06,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:57:06,356 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:57:10,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:10,934 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:57:15,936 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:20,937 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:21,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:57:21,356 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:57:21,356 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:57:26,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:31,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:36,412 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:36,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:57:36,470 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:57:36,470 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:57:40,937 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:57:41,939 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:46,940 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:51,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:57:51,470 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:57:52,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:57:57,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:02,517 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:06,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:58:06,470 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:58:08,511 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:10,940 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:58:13,942 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:18,943 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:21,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:58:21,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:58:21,658 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:58:24,700 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:29,703 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:34,703 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:36,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:58:36,658 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:58:39,709 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:40,943 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:58:44,946 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:49,947 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:58:51,657 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:58:51,658 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:58:55,709 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:00,710 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:05,711 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:06,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:59:06,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:59:06,896 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:59:10,935 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:10,946 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:59:15,948 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:20,949 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:21,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:59:21,896 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:59:21,896 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:59:26,935 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:31,936 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:36,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:59:36,896 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:59:36,896 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:59:37,935 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:40,949 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 19:59:42,951 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:47,952 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:51,895 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 19:59:51,896 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 19:59:51,896 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 19:59:53,940 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 19:59:58,941 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:03,943 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:06,896 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:00:06,896 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:00:06,897 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:00:08,954 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:10,953 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:00:13,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:18,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:21,947 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:00:21,948 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:00:21,948 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:00:24,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:29,007 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:34,008 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:37,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:00:37,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:00:37,151 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:00:39,202 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:40,955 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:00:44,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:49,958 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:00:52,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:00:52,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:00:52,151 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:00:55,203 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:00,205 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:05,206 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:07,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:01:07,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:01:07,151 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:01:10,957 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:01:10,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:15,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:20,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:22,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:01:22,151 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:01:22,151 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:01:26,200 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:31,201 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:36,203 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:37,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:01:37,151 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:01:37,151 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:01:40,960 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:01:41,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:46,963 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:51,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:01:52,150 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:01:52,151 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:01:52,151 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:01:57,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:02,217 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:07,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:02:07,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:02:07,168 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:02:08,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:10,963 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:02:13,966 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:18,967 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:22,151 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:02:22,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:02:22,168 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:02:24,224 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:29,225 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:34,227 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:37,151 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:02:37,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:02:37,168 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:02:39,235 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:40,966 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:02:44,969 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:49,970 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:02:52,151 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:02:52,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:02:52,168 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:02:55,210 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:00,211 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:05,212 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:07,235 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:03:07,235 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:03:07,236 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:03:10,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:10,969 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:03:15,972 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:20,973 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:22,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:03:22,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:03:22,237 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:03:26,292 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:31,293 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:36,295 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:37,235 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:03:37,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:03:37,236 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:03:40,972 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:03:41,974 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:46,975 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:51,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:03:52,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:03:52,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:03:52,237 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:03:57,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:02,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:07,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:04:07,236 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:04:07,237 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:04:07,293 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:10,975 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:04:12,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:17,978 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:22,369 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:04:22,370 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:04:22,371 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:04:23,413 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:28,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:33,415 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:37,369 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:04:37,370 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:04:38,430 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:40,978 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:04:43,980 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:48,981 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:52,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:04:52,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:04:52,400 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:04:54,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:04:59,446 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:04,447 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:07,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:05:07,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:05:07,400 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:05:10,447 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:10,982 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:05:15,985 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:20,986 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:22,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:05:22,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:05:22,401 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:05:26,470 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:31,471 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:36,472 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:37,560 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:05:37,561 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:05:37,561 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:05:40,984 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:05:41,987 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:46,988 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:51,989 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:05:52,560 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:05:52,561 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:05:52,561 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:05:57,599 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:02,600 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:07,601 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:07,694 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:06:07,736 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:06:07,737 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:06:10,987 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:06:12,989 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:17,991 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:22,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:06:22,992 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:23,020 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:06:23,021 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:06:28,070 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:33,072 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:37,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:06:37,978 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:06:37,978 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:06:39,034 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:40,990 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:06:44,993 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:49,994 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:06:52,983 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:06:52,984 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:06:52,985 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:06:55,024 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:00,025 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:05,027 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:07,983 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:07:07,984 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:07:07,984 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:07:10,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:10,993 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:07:15,996 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:20,997 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:23,762 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:07:23,763 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:07:23,763 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:07:26,804 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:31,805 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:36,807 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:38,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:07:38,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:07:38,966 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:07:40,996 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:07:41,998 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:46,999 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:52,000 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:07:53,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:07:53,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:07:53,966 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:07:57,011 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:02,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:07,013 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:08,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:08:08,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:08:08,966 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:08:10,999 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:08:13,001 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:18,002 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:23,003 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:23,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:08:23,966 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:08:23,966 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:08:29,002 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:34,003 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:38,967 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:08:39,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:39,009 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:08:39,009 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:08:41,002 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:08:45,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:50,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:08:53,967 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:08:53,967 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:08:53,968 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:08:55,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:00,013 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:05,013 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:09,616 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:09:09,616 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:09:09,617 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:09:10,689 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:11,004 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:09:16,007 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:21,008 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:25,206 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:09:25,207 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:09:25,207 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:09:26,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:31,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:36,276 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:40,206 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:09:40,206 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:09:40,207 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:09:41,007 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:09:42,009 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:47,010 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:52,011 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:09:55,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:09:55,231 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:09:55,231 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:09:57,285 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:02,287 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:07,288 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:10,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:10:10,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:10:10,231 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:10:11,011 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:10:13,013 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:18,014 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:23,015 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:25,272 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:10:25,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:10:25,273 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:10:28,331 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:33,332 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:38,334 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:40,272 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:10:40,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:10:40,273 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:10:41,013 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:10:44,015 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:49,016 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:54,017 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:10:55,272 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:10:55,273 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:10:59,311 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:04,312 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:09,313 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:10,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:11:10,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:11:10,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:11:11,016 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:11:15,019 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:20,020 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:25,021 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:25,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:11:25,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:11:25,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:11:30,351 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:35,352 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:40,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:11:40,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:11:41,019 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:11:41,024 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:46,023 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:51,024 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:11:55,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:11:55,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:11:56,348 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:01,350 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:06,351 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:10,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:12:10,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:12:11,022 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:12:12,024 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:17,026 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:22,027 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:25,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:12:25,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:12:25,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:12:28,012 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:33,013 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:38,014 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:40,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:12:40,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:12:41,025 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:12:43,027 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:48,028 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:53,029 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:12:55,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:12:55,274 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:12:58,327 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:03,329 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:08,330 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:10,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:13:10,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:13:11,028 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:13:14,030 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:19,032 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:24,033 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:25,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:13:25,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:13:29,367 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:34,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:39,369 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:40,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:13:40,276 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:13:40,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:13:41,031 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:13:45,034 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:50,035 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:55,036 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:13:55,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:13:55,275 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:14:00,317 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:05,318 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:10,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:14:10,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:14:10,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:11,034 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:14:16,036 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:21,038 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:25,276 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:14:25,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:14:26,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:31,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:36,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:40,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:14:40,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:14:41,038 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:14:42,040 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:47,041 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:52,042 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:14:55,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:14:55,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:14:55,277 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:14:57,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:02,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:07,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:10,276 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:15:10,276 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:15:11,040 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:15:13,041 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:18,043 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:23,044 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:25,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:15:25,278 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:15:28,357 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:33,358 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:38,359 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:40,276 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:15:40,277 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:15:41,043 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:15:44,045 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:49,046 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:54,047 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:15:55,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:15:55,277 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:15:59,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:04,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:09,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:10,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:16:10,277 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:16:11,046 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:16:15,048 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:20,049 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:25,050 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:25,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:16:25,278 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:16:30,327 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:35,329 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:40,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:16:40,278 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:16:40,333 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:41,049 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:16:46,051 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:51,052 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:16:55,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:16:55,278 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:16:56,314 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:01,315 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:06,317 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:10,379 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:17:10,420 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:17:10,421 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:17:11,052 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:17:12,054 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:17,055 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:22,056 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:25,380 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:17:25,380 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:17:25,381 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:17:27,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:32,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:37,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:40,379 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:17:40,380 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:17:40,380 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:17:41,055 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:17:43,057 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:48,059 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:53,060 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:17:55,542 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:17:55,542 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:17:55,543 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:17:58,584 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:03,585 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:08,586 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:10,542 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:18:10,543 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:18:10,543 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:18:11,058 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:18:14,060 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:19,062 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:24,063 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:25,543 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:18:25,543 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:18:25,544 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:18:29,591 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:34,592 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:39,593 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:40,726 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:18:40,726 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:18:40,727 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:18:41,061 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:18:45,063 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:50,065 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:55,065 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:18:55,725 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:18:55,726 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:18:55,727 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:19:00,788 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:05,789 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:10,726 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:19:10,726 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:19:10,727 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:19:11,064 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:19:11,067 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:16,067 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:21,068 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:25,768 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:19:25,768 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:19:25,769 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:19:26,849 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:31,851 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:36,852 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:40,769 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:19:40,769 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:19:40,770 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:19:41,068 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:19:42,070 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:47,072 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:52,073 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:19:55,768 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:19:55,769 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:19:57,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:02,814 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:07,816 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:10,769 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:20:10,769 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:20:11,070 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:20:13,072 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:18,073 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:23,075 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:25,959 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:20:25,959 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:20:25,960 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:20:29,004 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:34,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:39,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:40,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:20:40,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:20:40,961 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:20:41,073 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:20:44,075 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:49,076 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:54,077 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:20:55,959 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:20:55,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:20:55,960 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:21:00,001 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:05,002 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:10,003 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:10,959 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:21:10,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:21:10,960 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:21:11,076 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:21:15,079 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:20,080 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:25,081 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:25,959 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:21:25,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:21:25,961 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:21:31,051 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:36,052 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:41,053 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:41,079 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:21:41,359 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:21:41,360 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:21:41,360 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:21:46,405 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:51,406 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:21:56,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:01,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:06,688 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:07,987 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:22:07,988 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:22:07,988 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:22:11,082 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:22:12,084 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:17,085 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:22,086 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:22,987 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:22:22,988 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:22:22,988 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:22:28,027 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:33,029 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:37,987 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:22:37,988 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:22:37,988 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:22:38,040 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:41,085 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:22:43,087 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:48,088 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:53,089 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:22:53,141 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:22:53,142 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:22:53,142 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:22:58,196 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:03,197 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:08,198 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:08,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:23:08,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:23:08,306 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:23:11,088 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:23:14,090 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:19,092 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:23,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:23:23,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:23:23,307 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:23:24,354 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:29,356 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:34,356 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:38,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:23:38,307 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:23:38,307 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:23:40,348 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:41,091 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:23:46,094 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:51,095 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:23:53,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:23:53,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:23:53,307 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:23:56,366 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:01,367 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:06,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:08,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:24:08,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:24:08,307 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:24:11,094 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:24:12,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:17,097 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:22,098 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:23,789 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:24:23,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:24:23,790 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:24:27,869 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:32,870 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:37,872 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:38,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:24:38,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:24:38,790 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:24:41,098 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:24:43,101 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:48,102 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:53,103 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:24:53,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:24:53,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:24:53,790 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:24:58,840 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:03,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:08,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:25:08,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:25:08,791 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:25:08,858 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:11,100 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:25:14,102 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:19,103 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:23,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:25:23,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:25:23,791 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:25:24,836 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:29,837 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:34,838 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:39,321 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:25:39,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:25:39,322 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:25:40,377 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:41,103 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:25:46,106 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:51,107 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:25:54,321 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:25:54,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:25:54,323 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:25:56,373 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:01,375 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:06,376 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:09,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:26:09,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:26:09,323 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:26:11,106 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:26:12,108 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:17,109 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:22,110 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:24,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:26:24,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:26:24,323 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:26:27,367 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:32,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:37,369 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:39,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:26:39,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:26:39,323 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:26:41,109 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:26:43,111 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:48,112 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:53,113 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:26:54,322 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:26:54,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:26:54,323 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:26:58,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:03,388 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:08,389 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:09,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:27:09,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:27:09,324 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:27:11,112 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:27:14,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:19,115 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:24,116 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:24,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:27:24,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:27:24,324 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:27:29,371 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:34,372 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:39,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:27:39,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:27:39,324 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:27:39,379 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:41,115 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:27:45,118 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:50,119 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:27:54,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:27:54,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:27:54,324 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:27:55,374 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:00,375 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:05,376 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:09,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:28:09,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:28:09,324 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:28:10,378 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:11,118 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:28:16,121 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:21,122 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:24,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:28:24,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:28:24,324 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:28:26,370 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:31,371 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:36,373 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:39,324 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:28:39,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:28:39,325 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:28:41,121 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:28:42,123 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:47,124 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:52,125 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:28:54,484 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:28:54,484 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:28:54,485 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:28:57,546 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:02,547 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:07,548 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:09,484 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:29:09,484 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:29:09,485 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:29:11,125 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:29:13,127 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:18,128 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:23,130 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:24,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:29:24,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:29:24,917 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:29:29,003 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:34,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:39,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:39,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:29:39,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:29:39,917 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:29:41,127 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:29:44,129 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:49,130 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:54,131 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:29:54,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:29:54,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:29:54,917 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:29:59,979 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:04,980 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:09,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:30:09,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:30:09,917 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:30:10,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:11,130 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:30:16,132 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:21,133 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:24,916 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:30:24,917 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:30:26,965 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:31,966 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:36,968 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:39,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:30:39,918 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:30:39,918 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:30:41,132 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:30:42,134 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:47,136 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:52,137 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:30:54,917 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:30:54,918 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:30:57,978 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:02,979 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:07,981 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:10,154 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:31:10,154 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:31:10,155 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:31:11,136 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:31:13,138 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:18,140 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:23,141 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:25,724 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:31:25,724 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:31:25,725 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:31:28,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:33,841 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:38,843 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:40,724 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:31:40,724 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:31:40,725 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:31:41,139 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:31:44,141 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:49,142 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:54,143 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:31:55,756 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:31:55,757 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:31:55,757 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:31:59,799 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:04,800 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:09,801 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:10,756 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:32:10,757 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:32:10,757 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:32:11,142 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:32:15,144 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:20,145 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:25,146 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:25,935 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:32:25,936 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:32:25,936 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:32:30,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:35,978 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:40,979 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:41,016 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:32:41,016 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:32:41,017 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:32:41,145 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:32:46,147 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:51,148 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:32:56,016 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:32:56,017 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:32:57,061 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:02,062 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:07,064 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:11,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:33:11,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:33:11,115 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:33:11,160 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:33:12,162 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:17,163 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:22,164 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:26,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:33:26,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:33:26,115 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:33:27,184 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:32,185 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:37,186 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:41,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:33:41,114 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:33:41,115 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:33:41,163 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:33:43,165 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:48,166 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:53,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:33:56,762 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:33:56,762 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:33:56,763 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:33:58,814 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:03,815 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:08,816 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:11,154 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:34:11,762 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:34:11,762 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:34:11,763 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:34:14,806 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:19,807 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:24,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:27,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:34:27,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:34:27,097 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:34:30,146 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:35,147 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:40,148 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:41,159 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:34:42,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:34:42,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:34:42,097 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:34:46,145 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:51,146 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:56,147 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:34:57,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:34:57,096 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:34:57,097 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:35:01,661 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:06,662 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:11,159 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:35:12,135 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:35:12,135 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:35:12,136 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:35:12,179 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:17,180 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:22,181 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:27,135 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:35:27,136 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:35:27,136 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:35:27,205 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:32,206 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:37,207 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:41,164 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:35:42,135 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:35:42,136 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:35:42,137 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:35:43,182 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:48,183 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:53,184 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:35:57,135 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:35:57,136 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:35:57,136 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:35:58,188 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:03,189 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:08,191 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:11,166 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:36:12,135 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:36:12,136 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:36:12,137 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:36:14,189 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:19,191 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:24,192 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:27,228 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:36:27,228 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:36:27,229 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:36:29,282 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:34,283 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:39,284 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:41,169 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:36:42,228 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:36:42,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:36:42,229 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:36:45,272 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:50,273 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:55,274 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:36:57,228 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:36:57,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:36:57,229 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:37:00,286 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:05,288 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:10,289 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:11,172 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:37:12,228 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:37:12,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:37:12,229 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:37:16,269 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:21,271 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:26,272 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:27,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:37:27,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:37:27,230 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:37:31,308 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:36,309 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:41,175 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:37:42,177 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:42,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:37:42,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:37:42,230 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:37:47,269 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:52,270 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:57,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:37:57,271 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:37:57,272 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:37:57,273 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:38:02,337 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:07,339 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:11,178 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:38:12,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:38:12,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:38:12,230 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:38:13,278 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:18,279 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:23,280 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:27,229 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:38:27,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:38:27,231 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:38:29,280 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:34,281 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:39,283 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:41,181 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:38:42,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:38:42,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:38:42,395 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:38:44,468 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:49,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:54,471 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:38:57,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:38:57,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:38:57,395 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:39:00,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:05,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:10,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:11,185 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:39:12,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:39:12,438 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:39:12,439 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:39:15,488 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:20,489 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:25,490 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:27,514 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:39:27,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:39:27,516 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:39:30,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:35,554 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:40,555 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:41,186 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:39:42,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:39:42,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:39:42,517 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:39:45,582 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:50,583 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:55,584 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:39:57,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:39:57,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:39:57,516 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:40:01,580 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:06,581 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:11,189 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:40:12,191 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:12,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:40:12,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:40:12,517 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:40:17,638 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:22,640 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:27,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:40:27,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:40:27,517 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:40:28,556 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:33,557 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:38,558 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:41,192 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:40:43,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:40:43,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:40:43,007 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:40:44,046 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:49,050 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:54,050 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:40:58,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:40:58,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:40:58,007 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:40:59,066 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:04,068 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:09,069 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:11,196 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:41:13,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:41:13,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:41:13,006 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:41:15,044 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:20,045 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:25,046 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:28,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:41:28,048 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:41:28,049 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:41:30,119 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:35,120 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:40,121 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:41,198 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:41:43,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:41:43,007 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:41:43,008 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:41:46,063 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:51,064 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:56,065 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:41:58,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:41:58,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:41:58,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:42:01,444 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:06,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:11,201 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:42:12,203 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:13,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:42:13,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:42:13,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:42:17,429 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:22,430 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:27,431 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:28,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:42:28,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:42:28,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:42:32,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:37,438 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:41,204 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:42:43,207 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:43,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:42:43,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:42:43,385 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:42:48,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:53,435 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:42:58,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:42:58,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:42:58,385 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:42:59,431 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:04,432 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:09,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:11,207 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:43:13,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:43:13,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:43:15,431 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:20,432 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:25,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:28,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:43:28,385 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:43:31,429 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:36,430 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:41,210 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:43:42,212 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:43,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:43:43,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:43:43,745 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:43:47,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:52,804 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:57,805 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:43:58,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:43:58,744 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:44:02,812 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:07,813 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:11,215 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:44:13,217 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:13,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:44:13,744 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:44:18,800 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:23,801 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:28,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:44:28,745 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:44:29,791 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:34,792 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:39,793 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:41,216 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:44:43,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:44:43,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:44:43,914 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:44:44,975 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:49,976 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:54,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:44:58,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:44:58,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:44:58,914 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:45:00,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:05,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:10,963 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:11,219 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:45:13,912 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:45:13,913 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:45:16,952 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:21,954 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:26,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:28,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:45:28,913 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:45:31,987 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:36,988 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:41,222 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:45:42,224 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:43,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:45:43,914 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:45:43,914 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:45:47,960 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:52,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:57,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:45:58,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:45:58,914 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:46:03,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:08,963 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:11,225 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:46:13,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:46:13,914 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:46:14,961 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:19,962 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:24,963 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:28,913 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:46:28,914 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:46:29,981 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:34,982 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:39,983 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:41,228 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:46:44,195 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:46:44,195 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:46:44,196 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:46:45,238 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:50,239 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:55,241 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:46:59,195 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:46:59,196 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:46:59,196 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:47:00,244 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:05,245 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:10,247 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:11,231 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:47:14,294 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:47:14,294 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:47:14,294 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:47:15,334 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:20,335 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:25,336 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:29,294 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:47:29,294 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:47:29,295 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:47:30,380 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:35,381 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:40,382 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:41,234 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:47:44,295 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:47:44,295 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:47:44,295 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:47:46,371 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:51,372 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:56,373 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:47:59,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:47:59,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:47:59,364 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:48:01,403 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:06,404 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:11,238 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:48:12,241 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:14,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:48:14,363 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:48:17,434 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:22,435 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:27,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:29,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:48:29,364 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:48:32,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:37,438 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:41,242 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:48:43,245 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:44,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:48:44,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:48:44,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:48:48,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:53,438 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:58,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:48:59,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:48:59,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:48:59,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:49:04,439 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:09,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:11,244 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:49:14,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:49:14,383 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:49:15,422 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:20,423 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:25,424 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:29,382 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:49:29,383 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:49:30,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:35,446 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:40,447 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:41,247 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:49:44,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:49:44,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:49:44,384 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:49:45,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:50,453 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:55,454 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:49:59,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:49:59,385 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:50:01,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:06,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:11,251 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:50:12,253 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:14,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:50:14,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:50:14,709 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:50:17,759 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:22,760 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:27,761 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:29,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:50:29,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:50:29,709 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:50:33,754 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:38,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:41,253 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:50:44,256 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:44,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:50:44,709 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:50:44,709 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:50:49,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:54,756 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:50:59,757 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:00,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:51:00,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:51:00,341 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:51:05,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:10,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:11,256 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:51:15,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:51:15,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:51:15,341 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:51:15,397 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:20,398 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:25,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:30,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:51:30,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:51:30,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:51:31,396 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:36,398 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:41,260 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:51:42,262 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:45,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:51:45,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:51:45,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:51:47,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:52,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:51:57,402 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:00,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:52:00,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:52:00,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:52:02,417 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:07,418 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:11,263 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:52:13,265 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:15,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:52:15,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:52:15,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:52:18,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:23,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:28,401 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:30,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:52:30,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:52:34,391 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:39,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:41,265 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:52:45,268 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:45,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:52:45,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:52:50,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:52:55,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:00,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:53:00,343 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:53:00,386 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:05,387 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:10,388 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:11,269 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:53:15,341 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:53:15,342 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:53:16,381 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:21,382 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:26,383 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:30,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:53:30,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:53:30,369 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:53:31,431 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:36,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:41,273 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:53:42,275 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:45,368 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:53:45,369 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:53:45,369 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:53:47,430 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:52,431 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:53:57,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:00,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:54:00,408 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:54:00,408 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:54:02,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:07,478 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:11,275 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:54:13,277 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:15,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:54:15,408 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:54:15,408 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:54:18,457 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:23,458 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:28,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:30,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:54:30,408 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:54:30,409 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:54:34,457 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:39,458 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:41,277 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:54:45,280 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:46,391 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:54:46,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:54:46,392 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:54:50,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:54:55,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:00,445 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:01,391 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:55:01,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:55:01,393 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:55:05,450 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:10,451 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:11,281 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:55:16,284 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:16,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:55:16,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:55:16,393 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:55:21,465 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:26,466 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:31,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:55:31,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:55:31,393 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:55:32,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:37,449 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:41,284 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:55:43,286 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:46,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:55:46,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:55:46,393 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:55:48,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:53,449 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:55:58,450 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:01,413 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:56:01,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:56:01,414 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:56:06,603 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:56:07,112 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:11,287 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:56:12,289 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:16,413 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:56:16,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:56:16,414 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:56:17,457 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:22,458 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:27,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:31,413 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:56:31,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:56:31,414 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:56:33,460 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:38,461 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:41,290 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:56:44,292 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:46,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:56:46,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:56:46,415 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:56:49,456 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:54,457 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:56:59,458 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:01,774 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:57:01,774 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:57:01,775 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:57:04,824 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:09,825 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:11,294 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:57:15,297 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:16,775 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:57:16,775 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:57:16,776 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:57:20,819 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:25,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:30,821 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:31,773 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:57:31,774 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:57:35,840 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:40,842 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:41,296 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:57:46,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:46,773 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:57:46,774 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:57:51,814 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:57:56,815 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:01,774 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:58:01,775 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:58:01,775 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:58:01,818 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:06,819 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:11,300 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:58:12,302 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:16,781 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:58:16,781 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:58:16,781 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:58:17,828 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:22,830 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:27,831 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:31,780 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:58:31,781 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:58:31,781 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:58:32,831 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:37,832 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:41,302 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:58:43,304 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:47,159 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:58:47,159 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:58:47,160 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:58:49,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:54,217 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:58:59,218 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:02,348 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:59:02,348 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:59:02,349 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:59:04,395 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:09,397 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:11,305 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:59:15,307 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:17,347 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:59:17,348 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:59:20,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:25,395 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:30,396 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:32,583 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:59:32,583 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:59:32,584 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:59:35,651 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:40,652 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:41,308 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 20:59:46,310 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:47,583 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 20:59:47,583 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 20:59:47,584 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 20:59:51,641 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 20:59:56,642 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:01,643 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:02,701 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:00:02,702 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:00:02,702 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:00:06,753 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:11,312 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:00:12,315 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:17,316 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:17,702 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:00:17,702 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:00:17,702 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:00:22,751 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:27,752 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:32,701 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:00:32,702 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:00:32,771 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:37,772 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:41,315 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:00:43,317 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:47,701 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:00:47,702 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:00:48,745 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:53,746 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:00:58,747 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:02,702 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:01:02,702 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:01:03,749 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:08,750 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:11,318 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:01:14,320 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:17,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:01:17,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:01:17,706 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:01:19,741 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:24,742 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:29,743 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:32,704 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:01:32,705 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:01:32,748 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:01:34,779 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:39,781 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:41,321 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:01:45,323 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:47,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:01:47,705 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:01:47,748 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:01:50,788 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:01:55,789 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:00,790 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:02,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:02:02,706 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:02:02,748 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:02:06,758 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:11,324 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:02:12,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:17,327 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:17,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:02:17,748 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:02:17,749 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:02:22,796 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:27,798 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:32,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:02:32,706 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:02:38,005 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:38,102 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:02:41,327 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:02:43,328 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:47,705 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:02:47,706 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:02:48,757 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:53,758 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:02:58,759 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:02,706 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:03:02,706 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:03:02,707 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:03:04,749 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:09,750 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:11,330 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:03:15,332 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:17,707 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:03:17,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:03:17,709 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:03:20,764 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:25,765 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:30,767 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:32,706 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:03:32,707 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:03:36,747 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:41,332 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:03:42,334 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:47,335 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:47,752 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:03:47,752 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:03:47,753 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:03:52,819 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:03:57,820 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:02,752 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:04:02,753 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:04:02,753 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:04:03,803 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:08,804 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:11,335 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:04:14,337 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:17,753 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:04:17,753 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:04:17,754 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:04:19,806 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:24,808 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:29,809 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:33,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:04:33,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:04:33,306 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:04:35,358 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:40,359 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:41,338 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:04:46,340 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:48,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:04:48,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:04:48,306 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:04:51,349 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:04:56,351 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:01,352 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:03,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:05:03,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:05:03,306 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:05:06,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:11,340 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:05:12,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:17,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:18,305 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:05:18,306 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:05:18,306 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:05:23,342 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:28,343 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:33,344 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:33,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:05:33,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:05:33,437 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:05:38,486 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:41,343 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:05:44,345 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:48,395 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:05:48,395 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:05:48,395 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:05:49,435 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:54,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:05:59,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:03,395 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:06:03,396 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:06:03,396 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:06:04,440 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:09,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:11,346 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:06:15,348 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:18,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:06:18,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:06:18,476 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:06:20,508 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:25,510 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:30,511 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:33,474 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:06:33,475 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:06:35,540 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:40,541 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:41,349 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:06:46,352 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:48,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:06:48,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:06:48,476 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:06:51,521 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:06:56,522 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:01,524 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:03,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:07:03,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:07:03,476 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:07:07,522 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:11,352 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:07:13,353 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:18,355 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:18,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:07:18,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:07:18,477 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:07:23,514 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:28,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:33,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:07:33,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:07:33,476 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:07:33,530 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:38,531 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:41,355 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:07:44,357 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:48,475 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:07:48,476 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:07:49,527 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:54,528 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:07:59,529 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:03,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:08:03,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:08:03,477 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:08:04,531 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:09,532 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:11,359 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:08:15,361 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:18,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:08:18,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:08:18,478 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:08:20,515 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:25,516 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:30,518 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:33,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:08:33,477 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:08:37,708 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:41,361 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:08:43,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:48,364 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:48,476 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:08:48,477 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:08:53,539 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:08:58,540 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:03,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:09:03,477 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:09:04,524 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:09,525 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:11,364 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:09:15,366 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:18,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:09:18,478 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:09:18,478 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:09:20,526 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:25,527 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:30,528 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:33,624 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:09:33,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:09:33,625 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:09:35,724 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:40,725 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:41,367 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:09:46,370 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:48,624 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:09:48,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:09:48,625 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:09:51,665 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:09:56,667 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:01,668 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:03,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:10:03,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:10:03,626 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:10:07,667 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:11,370 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:10:13,372 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:18,373 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:18,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:10:18,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:10:18,626 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:10:23,673 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:28,674 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:33,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:10:33,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:10:33,626 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:10:33,693 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:38,694 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:41,373 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:10:44,375 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:48,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:10:48,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:10:48,626 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:10:49,678 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:54,679 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:10:59,680 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:03,625 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:11:03,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:11:03,626 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:11:05,663 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:10,664 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:11,376 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:11:16,379 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:18,633 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:11:18,676 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:11:18,677 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:11:21,720 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:26,722 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:31,723 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:33,633 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:11:33,634 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:11:33,634 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:11:36,727 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:41,379 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:11:42,381 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:47,382 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:49,068 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:11:49,069 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:11:49,069 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:11:53,111 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:11:58,112 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:03,113 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:04,069 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:12:04,069 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:12:04,070 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:12:08,120 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:11,382 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:12:13,384 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:18,385 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:19,069 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:12:19,070 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:12:19,071 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:12:24,119 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:29,120 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:34,122 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:34,710 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:12:34,711 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:12:34,712 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:12:39,773 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:41,385 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:12:45,387 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:49,710 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:12:49,711 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:12:49,711 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:12:50,755 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:12:55,756 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:00,757 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:04,720 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:13:04,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:13:04,721 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:13:05,764 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:10,765 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:11,389 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:13:16,392 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:19,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:13:19,721 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:13:19,722 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:13:21,766 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:26,767 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:31,768 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:34,745 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:13:34,745 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:13:34,746 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:13:36,794 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:41,391 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:13:42,393 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:47,394 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:49,898 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:13:49,898 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:13:49,899 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:13:52,944 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:13:57,946 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:02,947 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:04,898 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:14:04,898 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:14:04,899 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:14:07,951 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:11,394 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:14:13,396 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:18,397 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:20,245 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:14:20,245 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:14:20,246 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:14:24,299 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:29,300 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:34,301 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:35,244 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:14:35,245 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:14:35,245 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:14:40,292 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:41,396 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:14:45,399 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:50,400 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:14:50,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:14:50,626 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:14:50,627 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:14:55,680 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:00,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:05,683 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:05,956 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:15:05,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:15:05,957 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:15:11,004 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:11,399 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:15:16,402 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:20,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:15:20,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:15:20,958 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:15:21,998 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:26,999 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:32,000 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:35,956 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:15:35,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:15:35,957 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:15:37,999 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:41,402 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:15:43,404 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:48,406 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:50,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:15:50,957 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:15:54,031 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:15:59,032 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:04,035 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:05,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:16:05,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:16:05,958 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:16:10,008 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:11,405 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:16:15,408 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:20,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:20,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:16:20,958 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:16:25,998 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:30,999 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:35,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:16:35,958 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:16:35,958 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:16:36,002 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:41,003 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:41,408 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:16:46,411 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:50,957 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:16:50,958 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:16:52,001 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:16:57,002 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:02,003 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:05,958 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:17:05,958 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:17:07,996 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:11,411 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:17:13,413 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:18,414 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:20,958 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:17:20,959 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:17:20,959 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:17:24,004 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:29,006 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:34,007 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:36,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:17:36,168 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:17:36,168 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:17:39,207 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:41,416 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:17:44,418 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:49,419 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:17:51,167 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:17:51,168 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:17:51,168 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:17:55,230 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:00,231 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:05,232 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:06,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:18:06,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:18:06,554 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:18:10,406 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:11,417 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:18:15,419 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:20,420 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:21,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:18:21,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:18:21,554 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:18:25,594 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:30,596 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:35,597 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:36,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:18:36,554 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:18:36,554 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:18:40,601 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:41,420 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:18:46,423 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:51,424 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:18:51,553 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:18:51,554 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:18:51,554 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:18:56,603 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:01,604 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:06,605 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:06,681 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:19:06,681 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:19:06,682 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:19:11,423 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:19:12,425 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:17,426 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:21,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:19:21,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:19:21,683 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:19:22,733 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:27,735 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:32,736 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:36,681 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:19:36,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:19:36,682 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:19:38,730 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:41,426 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:19:44,428 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:49,429 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:51,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:19:51,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:19:51,683 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:19:54,731 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:19:59,732 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:04,733 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:06,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:20:06,682 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:20:06,683 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:20:10,720 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:11,429 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:20:16,432 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:21,433 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:22,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:20:22,200 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:20:22,201 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:20:27,246 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:32,247 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:37,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:20:37,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:20:37,161 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:20:38,211 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:41,433 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:20:43,435 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:48,436 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:52,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:20:52,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:20:52,161 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:20:54,214 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:20:59,215 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:04,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:07,160 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:21:07,161 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:21:10,205 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:11,435 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:21:15,437 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:20,439 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:22,162 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:21:22,162 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:21:22,163 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:21:26,215 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:31,216 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:36,217 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:37,161 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:21:37,162 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:21:41,438 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:21:41,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:46,441 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:51,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:21:52,161 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:21:52,162 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:21:57,214 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:02,215 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:07,161 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:22:07,162 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:22:08,208 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:11,441 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:22:13,442 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:18,443 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:22,162 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:22:22,163 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:22:22,164 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:22:24,201 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:29,202 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:34,204 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:37,162 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:22:37,163 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:22:37,163 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:22:39,209 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:41,445 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:22:44,447 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:49,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:22:52,189 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:22:52,190 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:22:52,190 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:22:55,240 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:00,241 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:05,242 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:07,189 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:23:07,190 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:23:07,190 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:23:11,233 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:11,447 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:23:16,448 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:21,449 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:22,191 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:23:22,191 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:23:22,192 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:23:27,232 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:32,233 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:37,234 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:37,939 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:23:37,939 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:23:37,940 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:23:41,450 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:23:42,452 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:47,454 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:52,454 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:23:52,939 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:23:52,940 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:23:57,992 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:02,993 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:07,994 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:08,317 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:24:08,318 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:24:08,318 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:24:11,453 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:24:13,455 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:18,456 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:23,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:24:23,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:24:23,410 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:24:23,460 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:28,461 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:33,462 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:38,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:38,737 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:24:38,737 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:24:38,738 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:24:41,456 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:24:44,459 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:49,460 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:53,737 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:24:53,737 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:24:54,831 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:24:59,832 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:04,833 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:08,738 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:25:08,738 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:25:08,739 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:25:10,784 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:11,459 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:25:16,461 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:21,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:23,738 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:25:23,738 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:25:23,739 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:25:26,776 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:31,778 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:36,779 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:38,737 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:25:38,738 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:25:41,462 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:25:42,463 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:47,464 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:52,465 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:25:53,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:25:53,744 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:25:53,745 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:25:57,796 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:02,797 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:07,798 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:08,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:26:08,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:26:08,956 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:26:11,465 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:26:13,467 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:18,468 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:23,469 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:23,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:26:23,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:26:23,955 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:26:29,000 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:34,001 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:38,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:26:38,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:26:38,955 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:26:39,994 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:41,468 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:26:45,470 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:50,471 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:26:53,954 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:26:53,955 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:26:56,001 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:01,001 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:06,002 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:08,955 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:27:08,956 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:27:08,956 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:27:11,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:11,471 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:27:16,473 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:21,474 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:24,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:27:24,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:27:24,326 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:27:27,418 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:32,419 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:37,420 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:39,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:27:39,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:27:39,326 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:27:41,475 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:27:42,477 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:47,478 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:52,479 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:27:54,325 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:27:54,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:27:54,326 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:27:58,375 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:03,376 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:08,377 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:09,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:28:09,326 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:28:09,327 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:28:11,476 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:28:13,479 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:18,480 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:23,481 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:24,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:28:24,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:28:24,363 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:28:29,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:34,410 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:39,361 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:28:39,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:28:39,363 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:28:40,409 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:41,479 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:28:45,482 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:50,483 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:28:54,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:28:54,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:28:54,363 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:28:56,403 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:01,404 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:06,405 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:09,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:29:09,364 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:29:09,364 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:29:11,482 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:29:11,487 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:16,485 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:21,487 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:24,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:29:24,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:29:24,364 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:29:27,405 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:32,406 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:37,407 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:39,362 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:29:39,363 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:29:39,364 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:29:41,485 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:29:42,487 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:47,488 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:52,490 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:29:54,890 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:29:54,891 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:29:54,891 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:29:57,940 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:02,941 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:07,942 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:09,890 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:30:09,891 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:30:09,891 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:30:11,488 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:30:13,490 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:18,491 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:23,493 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:24,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:30:24,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:30:24,893 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:30:28,931 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:33,932 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:38,933 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:39,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:30:39,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:30:39,893 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:30:41,490 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:30:44,492 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:49,494 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:54,495 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:30:54,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:30:54,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:30:54,893 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:30:59,940 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:04,941 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:09,893 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:31:09,893 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:31:09,894 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:31:09,954 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:11,493 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:31:15,495 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:20,496 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:24,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:31:24,893 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:31:24,893 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:31:25,976 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:30,977 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:35,978 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:39,892 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: keepalive +2023-07-11 21:31:39,893 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: stop_status +2023-07-11 21:31:39,894 DEBUG SenderThread:2302108 [sender.py:send_request():396] send_request: stop_status +2023-07-11 21:31:41,496 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:31:41,501 DEBUG HandlerThread:2302108 [handler.py:handle_request():144] handle_request: status_report +2023-07-11 21:31:42,379 INFO cpu :2302108 [interfaces.py:monitor():140] Process cpu has exited. +2023-07-11 21:31:42,380 DEBUG SystemMonitor:2302108 [system_monitor.py:_start():166] Finished system metrics aggregation loop +2023-07-11 21:31:42,381 DEBUG SystemMonitor:2302108 [system_monitor.py:_start():170] Publishing last batch of metrics +2023-07-11 21:31:42,386 DEBUG SenderThread:2302108 [sender.py:send():369] send: stats +2023-07-11 21:31:42,757 INFO MainThread:2302108 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/run-20230711_182238-06r7b4m3/logs/debug.log b/wandb/run-20230711_182238-06r7b4m3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..370842d895e566c5ef2acb2216f369815c573c0a --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/logs/debug.log @@ -0,0 +1,27 @@ +2023-07-11 18:22:38,531 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4 +2023-07-11 18:22:38,531 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Configure stats pid to 2301801 +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'} +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/logs/debug.log +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230711_182238-06r7b4m3/logs/debug-internal.log +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_init.py:init():547] calling init triggers +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_init.py:init():554] wandb.init called with sweep_config: {} +config: {} +2023-07-11 18:22:38,532 INFO MainThread:2301801 [wandb_init.py:init():596] starting backend +2023-07-11 18:22:38,533 INFO MainThread:2301801 [wandb_init.py:init():600] setting up manager +2023-07-11 18:22:38,535 INFO MainThread:2301801 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-07-11 18:22:38,539 INFO MainThread:2301801 [wandb_init.py:init():606] backend started and connected +2023-07-11 18:22:38,542 INFO MainThread:2301801 [wandb_init.py:init():703] updated telemetry +2023-07-11 18:22:38,543 INFO MainThread:2301801 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout +2023-07-11 18:22:40,245 INFO MainThread:2301801 [wandb_run.py:_on_init():2176] communicating current version +2023-07-11 18:22:40,322 INFO MainThread:2301801 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2023-07-11 18:22:40,322 INFO MainThread:2301801 [wandb_init.py:init():787] starting run threads in backend +2023-07-11 18:22:40,470 INFO MainThread:2301801 [wandb_run.py:_console_start():2155] atexit reg +2023-07-11 18:22:40,470 INFO MainThread:2301801 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW +2023-07-11 18:22:40,470 INFO MainThread:2301801 [wandb_run.py:_redirect():2075] Wrapping output streams. +2023-07-11 18:22:40,470 INFO MainThread:2301801 [wandb_run.py:_redirect():2100] Redirects installed. +2023-07-11 18:22:40,471 INFO MainThread:2301801 [wandb_init.py:init():828] run started, returning control to user process diff --git a/wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb b/wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..199d565f86f38e4580a51eb8cb1da077595f845a --- /dev/null +++ b/wandb/run-20230711_182238-06r7b4m3/run-06r7b4m3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9efbe5c7657be984345fe118e106fc277da1f713a6462629a49bba257c817235 +size 1247842 diff --git a/wandb/run-20230711_182238-k29fqnbt/files/config.yaml b/wandb/run-20230711_182238-k29fqnbt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..decef192026af7bd7d699d1be63a4a8065ccb098 --- /dev/null +++ b/wandb/run-20230711_182238-k29fqnbt/files/config.yaml @@ -0,0 +1,187 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.8.10 + cli_version: 0.15.4 + framework: huggingface + huggingface_version: 4.30.2 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1689114158.56903 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 71 + - 83 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 71 + - 83 + 3: + - 23 + - 24 + 4: 3.8.10 + 5: 0.15.4 + 6: 4.30.2 + 8: + - 2 + - 5 +pretrained_model_name_or_path: + desc: null + value: runwayml/stable-diffusion-v1-5 +revision: + desc: null + value: null +dataset_name: + desc: null + value: null +dataset_config_name: + desc: null + value: null +train_data_dir: + desc: null + value: /l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/ +image_column: + desc: null + value: image +caption_column: + desc: null + value: text +validation_prompt: + desc: null + value: tom cruise thumbs up +num_validation_images: + desc: null + value: 4 +validation_epochs: + desc: null + value: 1 +max_train_samples: + desc: null + value: null +output_dir: + desc: null + value: /l/vision/v5/sragas/easel_ai/models_plain/ +cache_dir: + desc: null + value: /l/vision/v5/sragas/hf_models/ +seed: + desc: null + value: 15 +resolution: + desc: null + value: 512 +center_crop: + desc: null + value: true +random_flip: + desc: null + value: true +train_batch_size: + desc: null + value: 2 +num_train_epochs: + desc: null + value: 300 +max_train_steps: + desc: null + value: 2400 +gradient_accumulation_steps: + desc: null + value: 4 +gradient_checkpointing: + desc: null + value: false +learning_rate: + desc: null + value: 1.0e-05 +scale_lr: + desc: null + value: false +lr_scheduler: + desc: null + value: cosine +lr_warmup_steps: + desc: null + value: 500 +snr_gamma: + desc: null + value: null +use_8bit_adam: + desc: null + value: false +allow_tf32: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.999 +adam_weight_decay: + desc: null + value: 0.01 +adam_epsilon: + desc: null + value: 1.0e-08 +max_grad_norm: + desc: null + value: 1.0 +push_to_hub: + desc: null + value: true +hub_token: + desc: null + value: null +prediction_type: + desc: null + value: null +hub_model_id: + desc: null + value: person-thumbs-up-plain-lora +logging_dir: + desc: null + value: logs +mixed_precision: + desc: null + value: null +report_to: + desc: null + value: wandb +local_rank: + desc: null + value: 0 +checkpointing_steps: + desc: null + value: 500 +checkpoints_total_limit: + desc: null + value: null +resume_from_checkpoint: + desc: null + value: null +enable_xformers_memory_efficient_attention: + desc: null + value: false +noise_offset: + desc: null + value: 0 +rank: + desc: null + value: 4 diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_3bac780bdad763dfe748.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_3bac780bdad763dfe748.png new file mode 100644 index 0000000000000000000000000000000000000000..7cf25d5ba284c83aeb93f7b1d04aeb16ee640f04 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_3bac780bdad763dfe748.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_3df8df281127a97b5d1a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_3df8df281127a97b5d1a.png new file mode 100644 index 0000000000000000000000000000000000000000..87a4845f8e7d920590339d0f740ac94aad34f3a0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_3df8df281127a97b5d1a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_fe76546257030d738699.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_fe76546257030d738699.png new file mode 100644 index 0000000000000000000000000000000000000000..28a78c82a34261b812b44269d36aff9ca40766ac Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_fe76546257030d738699.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_ffc1a9f27bcd6f2a61ac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_ffc1a9f27bcd6f2a61ac.png new file mode 100644 index 0000000000000000000000000000000000000000..2cb7f30b030e88f3eba921ed3719c805bce51c4a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1000_ffc1a9f27bcd6f2a61ac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_2e83a2626ff883d16244.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_2e83a2626ff883d16244.png new file mode 100644 index 0000000000000000000000000000000000000000..638b9d7f2f2fa9fcbdffbbbec00abc9a75cb2307 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_2e83a2626ff883d16244.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_386a62fe7e44ee995bbb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_386a62fe7e44ee995bbb.png new file mode 100644 index 0000000000000000000000000000000000000000..6ee8bd8394460a4bf125ccbebb3abcefa90fae97 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_386a62fe7e44ee995bbb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_808168a3292399e3b0a3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_808168a3292399e3b0a3.png new file mode 100644 index 0000000000000000000000000000000000000000..27d15213cd9a62863d0dd2d261f11b6a914bead6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_808168a3292399e3b0a3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_8bde6a294d23990b588f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_8bde6a294d23990b588f.png new file mode 100644 index 0000000000000000000000000000000000000000..509dfb73e659d57588f92660b5f5ccff0c23b45c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1008_8bde6a294d23990b588f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_219d5bff72b5592df3b9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_219d5bff72b5592df3b9.png new file mode 100644 index 0000000000000000000000000000000000000000..0c8a7f5b3d047aee759d721b278966339c214d1a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_219d5bff72b5592df3b9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_30d94ad44f6b1b049026.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_30d94ad44f6b1b049026.png new file mode 100644 index 0000000000000000000000000000000000000000..2d1dc0fc2ed0c3b8a3449370a3183d27f7e9599d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_30d94ad44f6b1b049026.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_49858d5caa724841d560.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_49858d5caa724841d560.png new file mode 100644 index 0000000000000000000000000000000000000000..94dab650b169b34c9665f18acfc198caa3a571d1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_49858d5caa724841d560.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_b2e26ce825c90767327d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_b2e26ce825c90767327d.png new file mode 100644 index 0000000000000000000000000000000000000000..c0342bd43f5f817162774fa3073c5789ccd374e6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1016_b2e26ce825c90767327d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_06f67fa746d45752b1c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_06f67fa746d45752b1c0.png new file mode 100644 index 0000000000000000000000000000000000000000..98fd5a7343fdccfd59a3c1dab1149c0c17aef7f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_06f67fa746d45752b1c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_78be3e36643b4f4e5aeb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_78be3e36643b4f4e5aeb.png new file mode 100644 index 0000000000000000000000000000000000000000..85e34e60a44019fc4f3e700e434c992b4b22dd5c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_78be3e36643b4f4e5aeb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_c8826549b51bc824f1a7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_c8826549b51bc824f1a7.png new file mode 100644 index 0000000000000000000000000000000000000000..c6e656a4086f8c75bd7c61f030507ebbda482257 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_c8826549b51bc824f1a7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_de6a26e8e064b3d4c07b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_de6a26e8e064b3d4c07b.png new file mode 100644 index 0000000000000000000000000000000000000000..fa071d4cd6328c1f11e52e379eff6c43c4e2ee4a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1024_de6a26e8e064b3d4c07b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_4440c94c8c5b21619372.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_4440c94c8c5b21619372.png new file mode 100644 index 0000000000000000000000000000000000000000..751493ae43617a055c1217b06e102a2bfc5c47f7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_4440c94c8c5b21619372.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_e0b5d6b0134b15f1acc9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_e0b5d6b0134b15f1acc9.png new file mode 100644 index 0000000000000000000000000000000000000000..5b7392fdcf550a8fb8009260f63a0afcb555f36f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_e0b5d6b0134b15f1acc9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_f343983400b249dec4cb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_f343983400b249dec4cb.png new file mode 100644 index 0000000000000000000000000000000000000000..fb3927e7a8ff4368275ee34c224c9edf751c14ef Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_f343983400b249dec4cb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_fa3287622e2f185b3f97.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_fa3287622e2f185b3f97.png new file mode 100644 index 0000000000000000000000000000000000000000..e1223e27b59831806f1d82eece6ea1f46913bae1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1032_fa3287622e2f185b3f97.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_0a890667226ffaf5b704.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_0a890667226ffaf5b704.png new file mode 100644 index 0000000000000000000000000000000000000000..d75ba61ebf87c945fe1633465f6f69bf1d372eaf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_0a890667226ffaf5b704.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_6fd15f77048c66e45ab2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_6fd15f77048c66e45ab2.png new file mode 100644 index 0000000000000000000000000000000000000000..c2f92697602ad06c5201e89239dbf6a60e1439a7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_6fd15f77048c66e45ab2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_a3486a6a586a730d4368.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_a3486a6a586a730d4368.png new file mode 100644 index 0000000000000000000000000000000000000000..2722adc0808a8033c6a42cf988394128b05bdc60 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_a3486a6a586a730d4368.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_d91b47aa5350d4e8bc34.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_d91b47aa5350d4e8bc34.png new file mode 100644 index 0000000000000000000000000000000000000000..c2f550eb61aed429cca51ec08c9ebf5d6be31eee Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1040_d91b47aa5350d4e8bc34.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_04bf65c9392f974c75b8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_04bf65c9392f974c75b8.png new file mode 100644 index 0000000000000000000000000000000000000000..a58b6c195a6050b7a89b4ebf20e365e4f4824f5e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_04bf65c9392f974c75b8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_8c9f97026c019506833a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_8c9f97026c019506833a.png new file mode 100644 index 0000000000000000000000000000000000000000..1dad28f6b755c3bb45b7763cb5f02037e4c16c44 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_8c9f97026c019506833a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_bd23a5c1898f8da42d93.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_bd23a5c1898f8da42d93.png new file mode 100644 index 0000000000000000000000000000000000000000..896951ffba4ecb91d7809daaa9f71794c655f819 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_bd23a5c1898f8da42d93.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_f14ff48f5c6ee9389577.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_f14ff48f5c6ee9389577.png new file mode 100644 index 0000000000000000000000000000000000000000..471a16084842108049f4851161997af0bfcbc138 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1048_f14ff48f5c6ee9389577.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_34b936bfe047d298afd0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_34b936bfe047d298afd0.png new file mode 100644 index 0000000000000000000000000000000000000000..2750f612ebbb724e32ca88c482e6761d33a48890 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_34b936bfe047d298afd0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_85d0efb91c0e48ceb8a6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_85d0efb91c0e48ceb8a6.png new file mode 100644 index 0000000000000000000000000000000000000000..2f1653295e69c1c3b835be39346f3429449b8c60 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_85d0efb91c0e48ceb8a6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_a3c6437c2be1111726ba.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_a3c6437c2be1111726ba.png new file mode 100644 index 0000000000000000000000000000000000000000..125e8a6d1c4933d6aae5ad75bc0b1286ecff8e33 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_a3c6437c2be1111726ba.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_d7a07b12de591dfc784b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_d7a07b12de591dfc784b.png new file mode 100644 index 0000000000000000000000000000000000000000..9c5ed49a6bec55d0f481b6c8591a1faedc66eb63 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_104_d7a07b12de591dfc784b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_0ac8c42ee2433dffe939.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_0ac8c42ee2433dffe939.png new file mode 100644 index 0000000000000000000000000000000000000000..59b58eec2ac19e976a8dfb233e26a69b69b8d056 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_0ac8c42ee2433dffe939.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_1479ab3105077bd77424.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_1479ab3105077bd77424.png new file mode 100644 index 0000000000000000000000000000000000000000..964c52f01be58202c77ae1ab6b041b6987f86d50 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_1479ab3105077bd77424.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_30c717b2b7e4a916ae89.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_30c717b2b7e4a916ae89.png new file mode 100644 index 0000000000000000000000000000000000000000..b95f2457a5461aea28b560c318273588cf9172e6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_30c717b2b7e4a916ae89.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_d16d0398bcc7dbf04bbf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_d16d0398bcc7dbf04bbf.png new file mode 100644 index 0000000000000000000000000000000000000000..9b6e1e821688f9a2d5da4c7c41c1aa4d397774f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1056_d16d0398bcc7dbf04bbf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_1cc5791fea4c65f0fde3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_1cc5791fea4c65f0fde3.png new file mode 100644 index 0000000000000000000000000000000000000000..bfcc89111e9740dcda131af99bfe246bdda194ed Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_1cc5791fea4c65f0fde3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_ae3016b396ea28947987.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_ae3016b396ea28947987.png new file mode 100644 index 0000000000000000000000000000000000000000..e8b294f7ae4540963b10cc5b1661c0f937600b05 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_ae3016b396ea28947987.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_b69fc5a0e25eb7503924.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_b69fc5a0e25eb7503924.png new file mode 100644 index 0000000000000000000000000000000000000000..240a38c19436b8ce574d2837252197214c6cd239 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_b69fc5a0e25eb7503924.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_e704101016c7c230c717.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_e704101016c7c230c717.png new file mode 100644 index 0000000000000000000000000000000000000000..c8c58082fca9d15184e8bcd82e7af79a6c971afd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1064_e704101016c7c230c717.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_3bd02124d49d0b8ff154.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_3bd02124d49d0b8ff154.png new file mode 100644 index 0000000000000000000000000000000000000000..2914a1e65d09cbfd46335c0e37f246e7988a48bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_3bd02124d49d0b8ff154.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_4118ab19403ed7a6f06d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_4118ab19403ed7a6f06d.png new file mode 100644 index 0000000000000000000000000000000000000000..b7e00ac465e3050b3314974a158909b8005e87c4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_4118ab19403ed7a6f06d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_41ce97897430def0ca20.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_41ce97897430def0ca20.png new file mode 100644 index 0000000000000000000000000000000000000000..b157fc176aeb05e220451c5e7d70392144bc8fbb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_41ce97897430def0ca20.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_fed73b1b21a38bd14ea8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_fed73b1b21a38bd14ea8.png new file mode 100644 index 0000000000000000000000000000000000000000..c562204f2921b7abc0bd01b15c703faa096ac6c8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1072_fed73b1b21a38bd14ea8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_1d776c3f3b5c17211029.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_1d776c3f3b5c17211029.png new file mode 100644 index 0000000000000000000000000000000000000000..49dbcfc83d5a1f478630161477e14fe69a67b762 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_1d776c3f3b5c17211029.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_5c4fecd9dcdd1e11f95a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_5c4fecd9dcdd1e11f95a.png new file mode 100644 index 0000000000000000000000000000000000000000..1a02d5765131cb3809f545830daf8a6ddb01d803 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_5c4fecd9dcdd1e11f95a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_9db4cb0a67fe9ae5cb6b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_9db4cb0a67fe9ae5cb6b.png new file mode 100644 index 0000000000000000000000000000000000000000..08ce5db97fe1d056d96b3e831b18375163d7de1d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_9db4cb0a67fe9ae5cb6b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_c32133924b5d95fd238b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_c32133924b5d95fd238b.png new file mode 100644 index 0000000000000000000000000000000000000000..af54d4ad57fcb49f5d8e6f5a514e8e8c6bc85fc8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1080_c32133924b5d95fd238b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_1676e030f39d2264623e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_1676e030f39d2264623e.png new file mode 100644 index 0000000000000000000000000000000000000000..a2dc2b759954eb1e7cd9d5ace6ef5a8cd076f85e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_1676e030f39d2264623e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_68176a9b6a649b7a461d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_68176a9b6a649b7a461d.png new file mode 100644 index 0000000000000000000000000000000000000000..cc350272f6fa1d16b3cc1823450d17ff39196a4c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_68176a9b6a649b7a461d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_cba58a2d69dc3368c094.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_cba58a2d69dc3368c094.png new file mode 100644 index 0000000000000000000000000000000000000000..a58cabb98cde6c576e95192e671e6d4a53109320 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_cba58a2d69dc3368c094.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_fd83b25b70ba4257c9b1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_fd83b25b70ba4257c9b1.png new file mode 100644 index 0000000000000000000000000000000000000000..0d8c47482f3cbae0310ca4cfb278ae70dfe1b753 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1088_fd83b25b70ba4257c9b1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_333e99e73a03c30fe42d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_333e99e73a03c30fe42d.png new file mode 100644 index 0000000000000000000000000000000000000000..388e0583da36d997cac7ad90aada7b717190cfee Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_333e99e73a03c30fe42d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_bd01b42ca6177ea47c5a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_bd01b42ca6177ea47c5a.png new file mode 100644 index 0000000000000000000000000000000000000000..c8ca57ed232bc8ea43fcdf430cb98f09829305e7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_bd01b42ca6177ea47c5a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_befbd5340fa0acbaf37e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_befbd5340fa0acbaf37e.png new file mode 100644 index 0000000000000000000000000000000000000000..7f832426ab5a0dd71c801fdf33d63ba5003c86f9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_befbd5340fa0acbaf37e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_c88fd5a40a05f4bac5c8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_c88fd5a40a05f4bac5c8.png new file mode 100644 index 0000000000000000000000000000000000000000..4f515e3d5a546e9c01eca902155113eb3c212749 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1096_c88fd5a40a05f4bac5c8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_1f83a28d25fff40729e7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_1f83a28d25fff40729e7.png new file mode 100644 index 0000000000000000000000000000000000000000..84533b17bc40914b8c1c7a686e3049432c1b2234 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_1f83a28d25fff40729e7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_637101446275b24ba9b1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_637101446275b24ba9b1.png new file mode 100644 index 0000000000000000000000000000000000000000..39b5e5771f9afc9ba20315920dc19813d0d6b0c9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_637101446275b24ba9b1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_bee308209e779d45bd8b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_bee308209e779d45bd8b.png new file mode 100644 index 0000000000000000000000000000000000000000..7918508633a71d27d4b0d984c9862d2e5c24fe61 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_bee308209e779d45bd8b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_d8dc6a37b8589618b062.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_d8dc6a37b8589618b062.png new file mode 100644 index 0000000000000000000000000000000000000000..e83399198358a33997abd8bdb390e07388ac3555 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1104_d8dc6a37b8589618b062.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_535e129af2802072858a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_535e129af2802072858a.png new file mode 100644 index 0000000000000000000000000000000000000000..ac863e4cc48a3354040ecbb2062d0874a7771e76 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_535e129af2802072858a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_9e12455b1c6a5c5317bb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_9e12455b1c6a5c5317bb.png new file mode 100644 index 0000000000000000000000000000000000000000..12f232ea2f3ac8002a9324a7e42da917e9854157 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_9e12455b1c6a5c5317bb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_e455625d1982cc47ad44.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_e455625d1982cc47ad44.png new file mode 100644 index 0000000000000000000000000000000000000000..bb36fa3a8f39ce50f90d613bb0892e398d5de010 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_e455625d1982cc47ad44.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_fb29bfacf919ea5e7503.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_fb29bfacf919ea5e7503.png new file mode 100644 index 0000000000000000000000000000000000000000..ad352d3692eac257303d61cd03ee11e825a423b4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1112_fb29bfacf919ea5e7503.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_09953f69ca2d54775598.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_09953f69ca2d54775598.png new file mode 100644 index 0000000000000000000000000000000000000000..881a4ea46bf49f7bb3468935d1187a6f14d8822b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_09953f69ca2d54775598.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_7383de8551e064f6ad43.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_7383de8551e064f6ad43.png new file mode 100644 index 0000000000000000000000000000000000000000..61f3190d5ed4ca8e24e1aef723be80296db0d7f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_7383de8551e064f6ad43.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_c92093d2b13b886281aa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_c92093d2b13b886281aa.png new file mode 100644 index 0000000000000000000000000000000000000000..672f3d727e0dc6caffb73baa27158b8861348ac5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_c92093d2b13b886281aa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_d29b97b60ac55d690e1b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_d29b97b60ac55d690e1b.png new file mode 100644 index 0000000000000000000000000000000000000000..9c21955974f09b39fbd672a709ab2355644a1016 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1120_d29b97b60ac55d690e1b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_0f91e55a8a98dd2e8ef8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_0f91e55a8a98dd2e8ef8.png new file mode 100644 index 0000000000000000000000000000000000000000..3a2f2cc9571bba47da02dd4a8dcd96bd929fe835 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_0f91e55a8a98dd2e8ef8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_3302c80927e3dde2eb3d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_3302c80927e3dde2eb3d.png new file mode 100644 index 0000000000000000000000000000000000000000..c2130ca16c39035af1bd1d5c99f82e6fb6abdf74 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_3302c80927e3dde2eb3d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_9459ebef2c19962e4b34.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_9459ebef2c19962e4b34.png new file mode 100644 index 0000000000000000000000000000000000000000..4c15cf44a20ca9bb95db192bc28d24f992f58b6c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_9459ebef2c19962e4b34.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_d8f656ea71301403f7bf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_d8f656ea71301403f7bf.png new file mode 100644 index 0000000000000000000000000000000000000000..fae93f6e46db850258839a6c8412663c75484d1a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1128_d8f656ea71301403f7bf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_37039f7d8b61bffb22b3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_37039f7d8b61bffb22b3.png new file mode 100644 index 0000000000000000000000000000000000000000..48accf5b7c40d6d1b5aac02c86148263a17d7c04 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_37039f7d8b61bffb22b3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_3994094417c23d4c4a21.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_3994094417c23d4c4a21.png new file mode 100644 index 0000000000000000000000000000000000000000..f19deca766082212b6ef6452a05bbd605fb93627 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_3994094417c23d4c4a21.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_58efbe7f01b6a5ce6ada.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_58efbe7f01b6a5ce6ada.png new file mode 100644 index 0000000000000000000000000000000000000000..b14d70a4df3d6bcd53549a38900e0b45643ca952 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_58efbe7f01b6a5ce6ada.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_77f47ee5ca4647b53e91.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_77f47ee5ca4647b53e91.png new file mode 100644 index 0000000000000000000000000000000000000000..198fa5f024ef7f909c1ca1e21cf1b407e63659a7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_112_77f47ee5ca4647b53e91.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_1b0f4331091bfdab39b6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_1b0f4331091bfdab39b6.png new file mode 100644 index 0000000000000000000000000000000000000000..d173d5dfde07baba8b5dea9c7d12a0b4a4d2b8c6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_1b0f4331091bfdab39b6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_6b3d6e01567109ccdd4f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_6b3d6e01567109ccdd4f.png new file mode 100644 index 0000000000000000000000000000000000000000..cbd1755044844d054ae38b54062f429b8f2fa4f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_6b3d6e01567109ccdd4f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_7271d56ca24970e2e44b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_7271d56ca24970e2e44b.png new file mode 100644 index 0000000000000000000000000000000000000000..14ff6cd430ac6c85b6a984b06c0041c0a9614749 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_7271d56ca24970e2e44b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_ed5070884043b0e9388b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_ed5070884043b0e9388b.png new file mode 100644 index 0000000000000000000000000000000000000000..bd4780d3398738a296cc60407126f4b00a66c0c9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1136_ed5070884043b0e9388b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_0f579e23872ee306cab9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_0f579e23872ee306cab9.png new file mode 100644 index 0000000000000000000000000000000000000000..bf65450586b19a06cefef7f353336dba24f3ff0f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_0f579e23872ee306cab9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_11f4342cbfa59cc9ac0e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_11f4342cbfa59cc9ac0e.png new file mode 100644 index 0000000000000000000000000000000000000000..913bca6a37fa9cb666b2ee42794134e52f9dec31 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_11f4342cbfa59cc9ac0e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_7debe473f9ee916b15a7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_7debe473f9ee916b15a7.png new file mode 100644 index 0000000000000000000000000000000000000000..caf7ad722ceec528a80c5c87d34d70b7d3db2599 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_7debe473f9ee916b15a7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_aeed065d00db7ef520a7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_aeed065d00db7ef520a7.png new file mode 100644 index 0000000000000000000000000000000000000000..76bbbe911a29a64b8febf8e1812e056ac6cd3f4b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1144_aeed065d00db7ef520a7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_75f582b863cdbbe4a504.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_75f582b863cdbbe4a504.png new file mode 100644 index 0000000000000000000000000000000000000000..bccb4ac891a941defb5996537fd58d385340d908 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_75f582b863cdbbe4a504.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_9ee0451c0db170f16218.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_9ee0451c0db170f16218.png new file mode 100644 index 0000000000000000000000000000000000000000..bbec29a7df108a8ed8437c26ee619378499f6002 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_9ee0451c0db170f16218.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_ac6be0e73608e31f23e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_ac6be0e73608e31f23e9.png new file mode 100644 index 0000000000000000000000000000000000000000..814974a82a8d3262e36e6cae85405a2f395e0c50 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_ac6be0e73608e31f23e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_f44e05a7e3e21014ac08.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_f44e05a7e3e21014ac08.png new file mode 100644 index 0000000000000000000000000000000000000000..2689ab0b62dec3686e85d59a1e61310834d08958 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1152_f44e05a7e3e21014ac08.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_2f1ca38ee1b0c5d85255.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_2f1ca38ee1b0c5d85255.png new file mode 100644 index 0000000000000000000000000000000000000000..775f47ad1a16f79c2cebb1c25fc851f83a78c7b6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_2f1ca38ee1b0c5d85255.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_b5862260f9dfb527cb9e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_b5862260f9dfb527cb9e.png new file mode 100644 index 0000000000000000000000000000000000000000..ab2a3345217698bf3fc406ddcf5185923baca953 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_b5862260f9dfb527cb9e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_bde0a03bf1ad8419d9c3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_bde0a03bf1ad8419d9c3.png new file mode 100644 index 0000000000000000000000000000000000000000..4473b890b8f346e70f05f945209a8f5dee156616 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_bde0a03bf1ad8419d9c3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_d1c53487db656ecbc8a8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_d1c53487db656ecbc8a8.png new file mode 100644 index 0000000000000000000000000000000000000000..02ff398e84fea8d50c39d2b35252dcef32419739 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1160_d1c53487db656ecbc8a8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_083b57a36fb9b48aa915.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_083b57a36fb9b48aa915.png new file mode 100644 index 0000000000000000000000000000000000000000..5c81d2383c37ac3b6f9a4d435f407e7383c46b47 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_083b57a36fb9b48aa915.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_7a5ab95838f1b5896fdb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_7a5ab95838f1b5896fdb.png new file mode 100644 index 0000000000000000000000000000000000000000..6b850a08135afa1f134e02dd496545f061694012 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_7a5ab95838f1b5896fdb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_d980d83f952d5c30a538.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_d980d83f952d5c30a538.png new file mode 100644 index 0000000000000000000000000000000000000000..d116feb7d0d0edf2db63da5528e04387828037f0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_d980d83f952d5c30a538.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_e6f6e202cb1d86be97bc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_e6f6e202cb1d86be97bc.png new file mode 100644 index 0000000000000000000000000000000000000000..1254defae129885a247a7c866704fba5b869c58c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1168_e6f6e202cb1d86be97bc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_b76e250604e885c4bd55.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_b76e250604e885c4bd55.png new file mode 100644 index 0000000000000000000000000000000000000000..2e696b0056b4bf813aaea9cd8e1dd409dda2e0f8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_b76e250604e885c4bd55.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_d028389a725dd737ece1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_d028389a725dd737ece1.png new file mode 100644 index 0000000000000000000000000000000000000000..b8ad5269bfb07ffcb89afc5d347c45d1e5bc68c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_d028389a725dd737ece1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_df0c9ae2823ce7d96461.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_df0c9ae2823ce7d96461.png new file mode 100644 index 0000000000000000000000000000000000000000..bfeea174ba72b3555f71d5e56095080f11f0437e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_df0c9ae2823ce7d96461.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_e97469ade650ac1ab331.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_e97469ade650ac1ab331.png new file mode 100644 index 0000000000000000000000000000000000000000..c25d1c4eddaaa3cfc32a33e48c206eb65d916c35 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1176_e97469ade650ac1ab331.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_05a4d2cabac98cd4d1d4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_05a4d2cabac98cd4d1d4.png new file mode 100644 index 0000000000000000000000000000000000000000..070128efc112b9dd49206e7ab6192a1e3037adfb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_05a4d2cabac98cd4d1d4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_09da65422971f700e437.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_09da65422971f700e437.png new file mode 100644 index 0000000000000000000000000000000000000000..2bd57c63af38351301990e9071fed3286de65e15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_09da65422971f700e437.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_974d5757846f0694ffb4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_974d5757846f0694ffb4.png new file mode 100644 index 0000000000000000000000000000000000000000..eba07fde579b9e1b4e50c8c8830fa9cdff167be8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_974d5757846f0694ffb4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_9984128afeb104db29a1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_9984128afeb104db29a1.png new file mode 100644 index 0000000000000000000000000000000000000000..7e828b45057e506665633fc1e1d431fc843d5fb5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1184_9984128afeb104db29a1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_0b8f02a4056f08fbfc01.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_0b8f02a4056f08fbfc01.png new file mode 100644 index 0000000000000000000000000000000000000000..0f722b5d7de2a9879556479de42781384ebf7155 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_0b8f02a4056f08fbfc01.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_14bd1888a41987672706.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_14bd1888a41987672706.png new file mode 100644 index 0000000000000000000000000000000000000000..61f49d2b595d2b2d02fc9b1af491a7c8bee41479 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_14bd1888a41987672706.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_62796f1f83e7ccdc060a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_62796f1f83e7ccdc060a.png new file mode 100644 index 0000000000000000000000000000000000000000..2f4bdbefc9906a137e6141bc32f93301acd7caaa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_62796f1f83e7ccdc060a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_d7bfbf7aea35b7bed028.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_d7bfbf7aea35b7bed028.png new file mode 100644 index 0000000000000000000000000000000000000000..d7c328a2b04e05fe2663e9ddfef59fee1bffa177 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1192_d7bfbf7aea35b7bed028.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_16f4a0c9ef346f446c13.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_16f4a0c9ef346f446c13.png new file mode 100644 index 0000000000000000000000000000000000000000..d667120b218a0c872f10e71ec6858f536bf3d844 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_16f4a0c9ef346f446c13.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_acc1bde7ddec3fe3577c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_acc1bde7ddec3fe3577c.png new file mode 100644 index 0000000000000000000000000000000000000000..d7b094eedc2afdd261ece3d08a3382edb4567460 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_acc1bde7ddec3fe3577c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_e93ddab6d31d607831c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_e93ddab6d31d607831c0.png new file mode 100644 index 0000000000000000000000000000000000000000..9beba84f30088bfeac1f04db6c378d433b8ffe83 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_e93ddab6d31d607831c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_edcc4e9f600fa42e7eff.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_edcc4e9f600fa42e7eff.png new file mode 100644 index 0000000000000000000000000000000000000000..6f7ef90328a989ead9136f7831ea6f1df3c7aa4e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1200_edcc4e9f600fa42e7eff.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_0a244457848c676d98e6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_0a244457848c676d98e6.png new file mode 100644 index 0000000000000000000000000000000000000000..8138aea5b85487611a8c3d595845261ee178ce30 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_0a244457848c676d98e6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_2403b0bdec188a248ed1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_2403b0bdec188a248ed1.png new file mode 100644 index 0000000000000000000000000000000000000000..2982d755fa7851098872545227a92facfae5c0cd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_2403b0bdec188a248ed1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_595eef34e4d21c6eb4a9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_595eef34e4d21c6eb4a9.png new file mode 100644 index 0000000000000000000000000000000000000000..2cd506728325e868f30599bf00a6abe6779b6484 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_595eef34e4d21c6eb4a9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_9fe00c3f196682ce8aed.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_9fe00c3f196682ce8aed.png new file mode 100644 index 0000000000000000000000000000000000000000..1dba1d6241d482d570289d03bb6bbf71234247c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1208_9fe00c3f196682ce8aed.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_10e52d49b55c29d4c70a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_10e52d49b55c29d4c70a.png new file mode 100644 index 0000000000000000000000000000000000000000..68b7c94e0eeca18c5f1646e40cc31979e9e3a09f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_10e52d49b55c29d4c70a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_70875a697c0992f9c209.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_70875a697c0992f9c209.png new file mode 100644 index 0000000000000000000000000000000000000000..549e81343dd20ce3b8708a11396220d42bf794a3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_70875a697c0992f9c209.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_bf2536cdbcf9021e038a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_bf2536cdbcf9021e038a.png new file mode 100644 index 0000000000000000000000000000000000000000..86d9c41d970efbef6736f08c8defef7aba1b559f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_bf2536cdbcf9021e038a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_e11e426abf94dc089311.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_e11e426abf94dc089311.png new file mode 100644 index 0000000000000000000000000000000000000000..ee779c0e1349cda773368f3842d764a03def9c30 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_120_e11e426abf94dc089311.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_0ce2aa47670fb06b9041.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_0ce2aa47670fb06b9041.png new file mode 100644 index 0000000000000000000000000000000000000000..b48786ebb37d8ec048c93c1f5db91b333fd27223 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_0ce2aa47670fb06b9041.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_6f69187eb5366064d111.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_6f69187eb5366064d111.png new file mode 100644 index 0000000000000000000000000000000000000000..bac9fff06c32219b31d4bef3e39733363777bf08 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_6f69187eb5366064d111.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_71a11268d66c7c3c3964.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_71a11268d66c7c3c3964.png new file mode 100644 index 0000000000000000000000000000000000000000..d4dd4c928d17d7addc9e8b5ceb13e3cb67dff3b0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_71a11268d66c7c3c3964.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_ce73942bbae6d5990516.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_ce73942bbae6d5990516.png new file mode 100644 index 0000000000000000000000000000000000000000..1e20b55a94dc5ea6094e30a6f504483e265d688d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1216_ce73942bbae6d5990516.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_2628069d13a5bea9e3df.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_2628069d13a5bea9e3df.png new file mode 100644 index 0000000000000000000000000000000000000000..80b697f33a1a9e44af876a67ceb825544571a32f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_2628069d13a5bea9e3df.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_39b1e250a34221277567.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_39b1e250a34221277567.png new file mode 100644 index 0000000000000000000000000000000000000000..94dca453134acac92d3eb8ecfbdcd33bd79adaf9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_39b1e250a34221277567.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_c5fa5e45c4aa753c17c9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_c5fa5e45c4aa753c17c9.png new file mode 100644 index 0000000000000000000000000000000000000000..3323abbfb960e9721aea4112360fc2dd3ada8fa0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_c5fa5e45c4aa753c17c9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_e37d195d179a05497868.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_e37d195d179a05497868.png new file mode 100644 index 0000000000000000000000000000000000000000..62cf87b980a45f9af83e3b2154da5511a2663ab5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1224_e37d195d179a05497868.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_4c55aad4840c63f78577.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_4c55aad4840c63f78577.png new file mode 100644 index 0000000000000000000000000000000000000000..870ed0a043c27df55e3552c6b932a28327746371 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_4c55aad4840c63f78577.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_87e2c345946d0460c998.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_87e2c345946d0460c998.png new file mode 100644 index 0000000000000000000000000000000000000000..04ca5e5596c6dd130edc2b25d6b91396a010724a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_87e2c345946d0460c998.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_d39d5a329f94acd4f6a4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_d39d5a329f94acd4f6a4.png new file mode 100644 index 0000000000000000000000000000000000000000..d3009ed491b9eff154c47df27b19920cf2792bec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_d39d5a329f94acd4f6a4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_e6497da0525aea382fbe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_e6497da0525aea382fbe.png new file mode 100644 index 0000000000000000000000000000000000000000..9a6506743b748d3a429e03af13a975233207f2db Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1232_e6497da0525aea382fbe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_4a6eef5fa6860d20cbc1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_4a6eef5fa6860d20cbc1.png new file mode 100644 index 0000000000000000000000000000000000000000..97a32e48ba62b30634f359a91bf451e2e52fad3c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_4a6eef5fa6860d20cbc1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_8441de0e32308471748a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_8441de0e32308471748a.png new file mode 100644 index 0000000000000000000000000000000000000000..8c1a2d4a95b7175b4cc13a01442fd8225c891482 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_8441de0e32308471748a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_9502bd66906821a48a2f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_9502bd66906821a48a2f.png new file mode 100644 index 0000000000000000000000000000000000000000..64c6ff7cd68f02ad147929d420a26ab53bd0f5b4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_9502bd66906821a48a2f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_c1ba7526a4470eb03bb6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_c1ba7526a4470eb03bb6.png new file mode 100644 index 0000000000000000000000000000000000000000..44bbbb5f87ffbbda69857239c379e97371d563c0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1240_c1ba7526a4470eb03bb6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_5c1a4d1cff1b24bda178.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_5c1a4d1cff1b24bda178.png new file mode 100644 index 0000000000000000000000000000000000000000..cb9a28fca45d6e4f703b3684dbe43ba57496b5d6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_5c1a4d1cff1b24bda178.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_d2b71281b721da5f3901.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_d2b71281b721da5f3901.png new file mode 100644 index 0000000000000000000000000000000000000000..5dd72d9eb81065aac34636a49959ce7332f88a98 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_d2b71281b721da5f3901.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_d8552d083a3c70763db2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_d8552d083a3c70763db2.png new file mode 100644 index 0000000000000000000000000000000000000000..1b2e479792bc797795c8fb46e3861055c0605df2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_d8552d083a3c70763db2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_f192f7502e6f7f5405e5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_f192f7502e6f7f5405e5.png new file mode 100644 index 0000000000000000000000000000000000000000..8c6bda7d344ed0fac84754dae930cecf0b545900 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1248_f192f7502e6f7f5405e5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_008ae0ebd6c28b34501c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_008ae0ebd6c28b34501c.png new file mode 100644 index 0000000000000000000000000000000000000000..98af279f59fc3591b2ee02168a3b4b6ea3bd845f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_008ae0ebd6c28b34501c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_4d5909a879197a9d6947.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_4d5909a879197a9d6947.png new file mode 100644 index 0000000000000000000000000000000000000000..c026a468d6dd55d98cf55ffbd2319542228f1c43 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_4d5909a879197a9d6947.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_6eaa59f4c776007b4577.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_6eaa59f4c776007b4577.png new file mode 100644 index 0000000000000000000000000000000000000000..9053242060402f4076ad0a05407d180b514e3c18 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_6eaa59f4c776007b4577.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_910aaa0aeb996f2bff52.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_910aaa0aeb996f2bff52.png new file mode 100644 index 0000000000000000000000000000000000000000..fd70669382a9024a42529d3c338b485e77636549 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1256_910aaa0aeb996f2bff52.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_12e521ee796cd4abe6b4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_12e521ee796cd4abe6b4.png new file mode 100644 index 0000000000000000000000000000000000000000..c02460817f9d04f08dda19d4c451f7fcd210d846 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_12e521ee796cd4abe6b4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_3c294db4afe2a101d5c2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_3c294db4afe2a101d5c2.png new file mode 100644 index 0000000000000000000000000000000000000000..283d12a8f524a255caf42e6f752269e95cb1abf3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_3c294db4afe2a101d5c2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_41d26f1ebd22f8010867.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_41d26f1ebd22f8010867.png new file mode 100644 index 0000000000000000000000000000000000000000..e507694b1a5f42843fb2fb86d1ea312f8c3e3761 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_41d26f1ebd22f8010867.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_a34f676857c79531f476.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_a34f676857c79531f476.png new file mode 100644 index 0000000000000000000000000000000000000000..1908699a46e6982d46428dec8f22f6f72c0a5ff0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1264_a34f676857c79531f476.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_0fe5a4f175505061d59f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_0fe5a4f175505061d59f.png new file mode 100644 index 0000000000000000000000000000000000000000..0801e17fae77fc5ca915e109d0e1e0fe3c93c06b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_0fe5a4f175505061d59f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_1d9217684c014aa87e19.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_1d9217684c014aa87e19.png new file mode 100644 index 0000000000000000000000000000000000000000..f21d01b3532670942fe400ca23201279d5491fdb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_1d9217684c014aa87e19.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_217b42b3f8761d16b428.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_217b42b3f8761d16b428.png new file mode 100644 index 0000000000000000000000000000000000000000..14d5c950578ac9d060d6e05439e444baba915179 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_217b42b3f8761d16b428.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_dd42d9becd27cc9d54fe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_dd42d9becd27cc9d54fe.png new file mode 100644 index 0000000000000000000000000000000000000000..0fe1bd5d41ed576a096cbca60370de23a3ad5979 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1272_dd42d9becd27cc9d54fe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_1cc9b7ef9b70d011ac71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_1cc9b7ef9b70d011ac71.png new file mode 100644 index 0000000000000000000000000000000000000000..a27507cd47cb629448fbaed6a4d3633505e6ecc6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_1cc9b7ef9b70d011ac71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_5c8b2ecfb16916c0242c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_5c8b2ecfb16916c0242c.png new file mode 100644 index 0000000000000000000000000000000000000000..1049afc51d08edac391a719b52f8686bf35ec846 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_5c8b2ecfb16916c0242c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_d62f57953ca77cb73b3f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_d62f57953ca77cb73b3f.png new file mode 100644 index 0000000000000000000000000000000000000000..50bc89b3db8ba27097dbcd4bf852ef7b1d61473f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_d62f57953ca77cb73b3f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_f43945339b235e0f01c6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_f43945339b235e0f01c6.png new file mode 100644 index 0000000000000000000000000000000000000000..30bffd9aa7c50531a6e22565363d3e47caf52924 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1280_f43945339b235e0f01c6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_401eb7d8a6e899c795fc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_401eb7d8a6e899c795fc.png new file mode 100644 index 0000000000000000000000000000000000000000..8496aeb7da1dc6a0774dea095014b53a112feab3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_401eb7d8a6e899c795fc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_477cc6da95a873dd86c6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_477cc6da95a873dd86c6.png new file mode 100644 index 0000000000000000000000000000000000000000..f81e6df83c1317a3418bb9da010bdbe8cb11c3ad Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_477cc6da95a873dd86c6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_ce6487d4a69d3d6621e0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_ce6487d4a69d3d6621e0.png new file mode 100644 index 0000000000000000000000000000000000000000..8ffb6bcfcadf92c8ee6e7b23fac278fd3f0662f8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_ce6487d4a69d3d6621e0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_d7f23e91466de6777b6a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_d7f23e91466de6777b6a.png new file mode 100644 index 0000000000000000000000000000000000000000..585ffb7e0455ac4d7b548a92a7ec0f80bc016f9d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1288_d7f23e91466de6777b6a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_c6d1e00286ea24ea39ba.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_c6d1e00286ea24ea39ba.png new file mode 100644 index 0000000000000000000000000000000000000000..6b62755b68ee52300e076bf653985c377d1b68a3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_c6d1e00286ea24ea39ba.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_c9fb58eadba85e11521d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_c9fb58eadba85e11521d.png new file mode 100644 index 0000000000000000000000000000000000000000..4c99903256ab57f2d42e1d3cfc24c1b158a00a9d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_c9fb58eadba85e11521d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_f12975048cb9e24609f6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_f12975048cb9e24609f6.png new file mode 100644 index 0000000000000000000000000000000000000000..6a48314993850cd125c5a94a65c722eeb6b53df9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_f12975048cb9e24609f6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_f4dfbac2cbc6682335d5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_f4dfbac2cbc6682335d5.png new file mode 100644 index 0000000000000000000000000000000000000000..57cb20460a77ca75978e6ec4ce3a78c2be53c3e1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_128_f4dfbac2cbc6682335d5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_23cc3b9e181168d6ed71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_23cc3b9e181168d6ed71.png new file mode 100644 index 0000000000000000000000000000000000000000..0aa03484ffa466851d4f9ff1a69024dbe92ac6ec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_23cc3b9e181168d6ed71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_2a52a9d5fd9d9e666456.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_2a52a9d5fd9d9e666456.png new file mode 100644 index 0000000000000000000000000000000000000000..e0a677935d39e48aeebe06fbb84ee8a1b9b151f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_2a52a9d5fd9d9e666456.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_42e08916c70b880ffc82.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_42e08916c70b880ffc82.png new file mode 100644 index 0000000000000000000000000000000000000000..aced62d06fe9dcbbd624846bfb8fce7a1c984ad3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_42e08916c70b880ffc82.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_db936d1e5ef08167a9b9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_db936d1e5ef08167a9b9.png new file mode 100644 index 0000000000000000000000000000000000000000..a6b0581317f0c0d281793dddf9e92f0a2fbd3309 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1296_db936d1e5ef08167a9b9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_1d25ec0bcdc8cc75bf15.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_1d25ec0bcdc8cc75bf15.png new file mode 100644 index 0000000000000000000000000000000000000000..dbf908d9aafc7fda617d4655b350ef028ce3f0ac Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_1d25ec0bcdc8cc75bf15.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_500c611fcafe0a66ed45.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_500c611fcafe0a66ed45.png new file mode 100644 index 0000000000000000000000000000000000000000..a4b9cb6148ad699cdde7dcd81a5c144c8b9df267 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_500c611fcafe0a66ed45.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_5455e556b43dc90874e7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_5455e556b43dc90874e7.png new file mode 100644 index 0000000000000000000000000000000000000000..9dbeeed32f97593d4e0b675625deb25e5249461b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_5455e556b43dc90874e7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_bb39219de51d2ce5bad3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_bb39219de51d2ce5bad3.png new file mode 100644 index 0000000000000000000000000000000000000000..bbc2a39aa27ec20564c69b594be829f32cd98cdf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1304_bb39219de51d2ce5bad3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_2f48e342607d482d7214.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_2f48e342607d482d7214.png new file mode 100644 index 0000000000000000000000000000000000000000..ceba873ae3c43778f7c6dbf8b5786581e0612b3d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_2f48e342607d482d7214.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_3ec5f179893340066c13.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_3ec5f179893340066c13.png new file mode 100644 index 0000000000000000000000000000000000000000..faba9b1458381297eca26eb6f098cb9b989bded3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_3ec5f179893340066c13.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_8123dc447f99405e7c54.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_8123dc447f99405e7c54.png new file mode 100644 index 0000000000000000000000000000000000000000..570e7aaccc4a0f6b57fd14577c33f3ed0721e227 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_8123dc447f99405e7c54.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_c4b40ef60ee62f599312.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_c4b40ef60ee62f599312.png new file mode 100644 index 0000000000000000000000000000000000000000..59c5241a5f673d1e21f7a6eed0fad7a4ace3ca99 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1312_c4b40ef60ee62f599312.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_808df0a2d3159d683778.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_808df0a2d3159d683778.png new file mode 100644 index 0000000000000000000000000000000000000000..5dc45002fb48e29f0292d4eb88f40f50b89cdd33 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_808df0a2d3159d683778.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_8cc1e0a2f932c625950d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_8cc1e0a2f932c625950d.png new file mode 100644 index 0000000000000000000000000000000000000000..3ba8cbbe100647b6706cbfd8c83e548f37179a03 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_8cc1e0a2f932c625950d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_9444d2a3d4d06dc520ad.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_9444d2a3d4d06dc520ad.png new file mode 100644 index 0000000000000000000000000000000000000000..8640a873dc44aa376c498940f7a585eb17434a4e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_9444d2a3d4d06dc520ad.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_bc6cc209488d5a0dd512.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_bc6cc209488d5a0dd512.png new file mode 100644 index 0000000000000000000000000000000000000000..d617ed2c764f766388142275bcda341185958024 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1320_bc6cc209488d5a0dd512.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_2a8e50b9a2f0ea3c709b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_2a8e50b9a2f0ea3c709b.png new file mode 100644 index 0000000000000000000000000000000000000000..3975ad650067948b011dafdc58823e858f0d1d6c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_2a8e50b9a2f0ea3c709b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_3644a270a89ab0e7f320.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_3644a270a89ab0e7f320.png new file mode 100644 index 0000000000000000000000000000000000000000..227ea0ba32aafd7f0d8de6e6e8a6661cbc0a98ca Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_3644a270a89ab0e7f320.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_66a0e97f0510c0041e71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_66a0e97f0510c0041e71.png new file mode 100644 index 0000000000000000000000000000000000000000..b824f81cc00ab4b606f5a6f77613b62943e15111 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_66a0e97f0510c0041e71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_d5ddf8aec98ceaf31a6d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_d5ddf8aec98ceaf31a6d.png new file mode 100644 index 0000000000000000000000000000000000000000..d7b6dc9d575c53bd983bcd12deb0b500c86e58ec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1328_d5ddf8aec98ceaf31a6d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_1c736decbf969e547f0e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_1c736decbf969e547f0e.png new file mode 100644 index 0000000000000000000000000000000000000000..cebbf6bf43554430e2468a1e4a8f493d32e464ec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_1c736decbf969e547f0e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_1e36f4db41c7e9a383a2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_1e36f4db41c7e9a383a2.png new file mode 100644 index 0000000000000000000000000000000000000000..1cdc4730b60ef53bbff82d413f6f5f97ccdc3921 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_1e36f4db41c7e9a383a2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_5c456d466b0c671052e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_5c456d466b0c671052e3.png new file mode 100644 index 0000000000000000000000000000000000000000..446786dc4d68eb046e89f591b2acaf3c580f17ab Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_5c456d466b0c671052e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_bd08d38f68d5827ddd30.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_bd08d38f68d5827ddd30.png new file mode 100644 index 0000000000000000000000000000000000000000..6ca283dce1a9589163a795891732d759a5a16693 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1336_bd08d38f68d5827ddd30.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_1b37e5405be2f6007744.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_1b37e5405be2f6007744.png new file mode 100644 index 0000000000000000000000000000000000000000..a2392c6f28fa489292a8c1a1dedef250be9e8a26 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_1b37e5405be2f6007744.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_365a301a6ed3b1ea3869.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_365a301a6ed3b1ea3869.png new file mode 100644 index 0000000000000000000000000000000000000000..ff14c0262c9d2e905c2a8da9c6659dc072c4c289 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_365a301a6ed3b1ea3869.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_48edd71e0c11119f6254.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_48edd71e0c11119f6254.png new file mode 100644 index 0000000000000000000000000000000000000000..2bbd578ac9d37621723b8d5e256cc33a736cb8ca Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_48edd71e0c11119f6254.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_fdc245de4607a1ec4488.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_fdc245de4607a1ec4488.png new file mode 100644 index 0000000000000000000000000000000000000000..a7cfa53097ec465b305571bf7f097f6b71864bc8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1344_fdc245de4607a1ec4488.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_4092798bf460bac8f7a8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_4092798bf460bac8f7a8.png new file mode 100644 index 0000000000000000000000000000000000000000..d35461f60fef7dc5aea85421dc21a2f12c45fba1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_4092798bf460bac8f7a8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_8fd1600004e6bf64927d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_8fd1600004e6bf64927d.png new file mode 100644 index 0000000000000000000000000000000000000000..a1be5f7bcb8e694c2c38f815796fe748d854d091 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_8fd1600004e6bf64927d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_9573be96ed0d456f9e97.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_9573be96ed0d456f9e97.png new file mode 100644 index 0000000000000000000000000000000000000000..19f815564dcaa66ccae1259f62a7ddabd1b736b7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_9573be96ed0d456f9e97.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_eca533871e5a572dfd60.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_eca533871e5a572dfd60.png new file mode 100644 index 0000000000000000000000000000000000000000..cae32423b30e1f1b7e2f8b1bb2c69f0464c9162e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1352_eca533871e5a572dfd60.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_130544406f3b3910ce09.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_130544406f3b3910ce09.png new file mode 100644 index 0000000000000000000000000000000000000000..fc8a0b436d6e909e74a2f4da3a95dfb6ad2f36c2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_130544406f3b3910ce09.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_1346866ddc9966bbc550.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_1346866ddc9966bbc550.png new file mode 100644 index 0000000000000000000000000000000000000000..e7c5b3c08b77a40b965bbad7c51cf3b3805bb55c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_1346866ddc9966bbc550.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_61604253d2d4fc950a9b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_61604253d2d4fc950a9b.png new file mode 100644 index 0000000000000000000000000000000000000000..217d53ff205dfaf1de8ad893633dad76b9a337bf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_61604253d2d4fc950a9b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_905182c9f1a3424d43d9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_905182c9f1a3424d43d9.png new file mode 100644 index 0000000000000000000000000000000000000000..8c47e74cbd7b60e8c78ee85f1fe5c557cac377f7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1360_905182c9f1a3424d43d9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_26163caf1003b87c0917.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_26163caf1003b87c0917.png new file mode 100644 index 0000000000000000000000000000000000000000..959c12e85fabf060d30591011c3b5b9987a4d39f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_26163caf1003b87c0917.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_8ccd1f2533b3ff46050a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_8ccd1f2533b3ff46050a.png new file mode 100644 index 0000000000000000000000000000000000000000..ba5c633ae1d6bbb5c62d3bad6df6ff4d384d6ff3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_8ccd1f2533b3ff46050a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_c0a22e845a09ff0cd3c5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_c0a22e845a09ff0cd3c5.png new file mode 100644 index 0000000000000000000000000000000000000000..d6ea40f59715de68034af6bb0cf067a0a388dc45 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_c0a22e845a09ff0cd3c5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_ed8be6871a61d2c59c72.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_ed8be6871a61d2c59c72.png new file mode 100644 index 0000000000000000000000000000000000000000..61144ce98775bee02f6adf6209398f0fd171b2fa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1368_ed8be6871a61d2c59c72.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_36bb53c164993a8cf8d8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_36bb53c164993a8cf8d8.png new file mode 100644 index 0000000000000000000000000000000000000000..ddca3efbe6a01e043df3e9ceae9403cfb4a6f1b3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_36bb53c164993a8cf8d8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_3b8a8548b12f4fe561b0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_3b8a8548b12f4fe561b0.png new file mode 100644 index 0000000000000000000000000000000000000000..ce8e862466d5dedb80808a171ec355a60c6f9cc7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_3b8a8548b12f4fe561b0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_4a6f2aa08c542f4f3d3b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_4a6f2aa08c542f4f3d3b.png new file mode 100644 index 0000000000000000000000000000000000000000..f72b3ef3e5d078bd9f1be2b2fe085e0d4dca25cb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_4a6f2aa08c542f4f3d3b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_e3e73b728661ef994342.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_e3e73b728661ef994342.png new file mode 100644 index 0000000000000000000000000000000000000000..621ff51ec2b15fa59dc5cc95a95c2353fed422e1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_136_e3e73b728661ef994342.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_1bed837a55c2e8de98f6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_1bed837a55c2e8de98f6.png new file mode 100644 index 0000000000000000000000000000000000000000..d406fef9f00218eb9262b27c824c1f639a3fdd7d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_1bed837a55c2e8de98f6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_9731a8706e6ed0046c32.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_9731a8706e6ed0046c32.png new file mode 100644 index 0000000000000000000000000000000000000000..3df96a1f89bdd1753504b5eeade718a583e25503 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_9731a8706e6ed0046c32.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_a1dbd4bbb6ca33baecc1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_a1dbd4bbb6ca33baecc1.png new file mode 100644 index 0000000000000000000000000000000000000000..4e390cbec3f28c0b65682e1d8be1a3a6eb2f8145 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_a1dbd4bbb6ca33baecc1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_a8c45d7ccc7b5c5f5a87.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_a8c45d7ccc7b5c5f5a87.png new file mode 100644 index 0000000000000000000000000000000000000000..a7885e4853eac9a5d89c05fdfcacd1d54dbdeba0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1376_a8c45d7ccc7b5c5f5a87.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_52b65704857359088b8a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_52b65704857359088b8a.png new file mode 100644 index 0000000000000000000000000000000000000000..b89c21096156d0f6770eb863a70716b44ca22503 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_52b65704857359088b8a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_91c90db7ff100b331d46.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_91c90db7ff100b331d46.png new file mode 100644 index 0000000000000000000000000000000000000000..81bd6705d8837fbebbe3a396f69965085fc34fd8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_91c90db7ff100b331d46.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_a0a8d6a6682689911e9c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_a0a8d6a6682689911e9c.png new file mode 100644 index 0000000000000000000000000000000000000000..5f8e3b848c5da8dd9399c2a18505003ce5d4ffd8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_a0a8d6a6682689911e9c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_ca0a6b436b4b86c17ede.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_ca0a6b436b4b86c17ede.png new file mode 100644 index 0000000000000000000000000000000000000000..b069f12c720041f2948ac82a67f68ddf4d153d15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1384_ca0a6b436b4b86c17ede.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_a4ad4ffc5207f63facbc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_a4ad4ffc5207f63facbc.png new file mode 100644 index 0000000000000000000000000000000000000000..97316ca538077d5029ee4bd94f6c6e08c900a183 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_a4ad4ffc5207f63facbc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_ca30155883656ac69b8a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_ca30155883656ac69b8a.png new file mode 100644 index 0000000000000000000000000000000000000000..c5e0ea340467a79e1208ba375a4fa1b174dd8ef9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_ca30155883656ac69b8a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_eca6ba6691c330e7631e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_eca6ba6691c330e7631e.png new file mode 100644 index 0000000000000000000000000000000000000000..cfbb8a02f43b4f10c29631962be1ca409e047d7f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_eca6ba6691c330e7631e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_f0607d0b692156045759.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_f0607d0b692156045759.png new file mode 100644 index 0000000000000000000000000000000000000000..d5cc3c9af5063d9b42adfa0590bbe8cf01d9aa84 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1392_f0607d0b692156045759.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_4c95c187c0312a8c87f2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_4c95c187c0312a8c87f2.png new file mode 100644 index 0000000000000000000000000000000000000000..58b5b654e107a6bdf2e27273696bbf9aee7901d4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_4c95c187c0312a8c87f2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_5bf1b64dcf28d3495823.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_5bf1b64dcf28d3495823.png new file mode 100644 index 0000000000000000000000000000000000000000..062f7e2e49affae7eb4689d115bb39d359b7d669 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_5bf1b64dcf28d3495823.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_8023481f74cd92744fdd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_8023481f74cd92744fdd.png new file mode 100644 index 0000000000000000000000000000000000000000..3c091c094ae1a1098f8bc96cebc0ef439f25702d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_8023481f74cd92744fdd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_a6f4367b333bbbefc831.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_a6f4367b333bbbefc831.png new file mode 100644 index 0000000000000000000000000000000000000000..c3c6ba6b198324e0c10bfbb3e8a698ac57acfa2e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1400_a6f4367b333bbbefc831.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_64af9f65d337fc4f7fc0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_64af9f65d337fc4f7fc0.png new file mode 100644 index 0000000000000000000000000000000000000000..1a0e7492cc8eceb1a8a6fd63eaf45386176a3d8c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_64af9f65d337fc4f7fc0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_673f682e941b2195ef5f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_673f682e941b2195ef5f.png new file mode 100644 index 0000000000000000000000000000000000000000..b529414310bed9a23a04ef2154187f058c654dc1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_673f682e941b2195ef5f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_a9dee32d07729bbe5ab6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_a9dee32d07729bbe5ab6.png new file mode 100644 index 0000000000000000000000000000000000000000..3e813cc2cffb3079c695e199caddda3696b256a4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_a9dee32d07729bbe5ab6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_d9062fb94fef4cfadbd2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_d9062fb94fef4cfadbd2.png new file mode 100644 index 0000000000000000000000000000000000000000..33dc667965d2284f97a8da0c6372f540a581d61c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1408_d9062fb94fef4cfadbd2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_3aead0846726e7166380.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_3aead0846726e7166380.png new file mode 100644 index 0000000000000000000000000000000000000000..c8f27ca35957845876c8ac7aaa0732e5a18af51c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_3aead0846726e7166380.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_42fdf771bb26d71377ce.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_42fdf771bb26d71377ce.png new file mode 100644 index 0000000000000000000000000000000000000000..d4d6a1d0c99875fc7dc981e9f975690fc3028910 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_42fdf771bb26d71377ce.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_8e95d8013f7d1cb53ee7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_8e95d8013f7d1cb53ee7.png new file mode 100644 index 0000000000000000000000000000000000000000..8089ab677ef37fddcad99ac91a0c944dc01819a3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_8e95d8013f7d1cb53ee7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_9a51711877bf807947e8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_9a51711877bf807947e8.png new file mode 100644 index 0000000000000000000000000000000000000000..5f95e198bb3a53c423d0ebe013ec14d344294e8c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1416_9a51711877bf807947e8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_53e42d50be50609b55e6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_53e42d50be50609b55e6.png new file mode 100644 index 0000000000000000000000000000000000000000..7f84c7f1149d154b6220d40bf32294bf7373a8f4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_53e42d50be50609b55e6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_7ce5c6417cd370810ef9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_7ce5c6417cd370810ef9.png new file mode 100644 index 0000000000000000000000000000000000000000..e0fd8b66515392430e50ed6ac3c06f0c9dac8311 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_7ce5c6417cd370810ef9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_ac49a027c2b42beeb91d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_ac49a027c2b42beeb91d.png new file mode 100644 index 0000000000000000000000000000000000000000..1e7c49294f8a9f8524ae23d7687f40989ce9dc0c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_ac49a027c2b42beeb91d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_bc80783a000c38fd4ffb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_bc80783a000c38fd4ffb.png new file mode 100644 index 0000000000000000000000000000000000000000..594819c9b89415e227d52c3fb4c36d8099639a39 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1424_bc80783a000c38fd4ffb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_43ab5d4df5c3f9be760e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_43ab5d4df5c3f9be760e.png new file mode 100644 index 0000000000000000000000000000000000000000..5bfd925fd9004612f52ce49d0f2844e4cf2c6f63 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_43ab5d4df5c3f9be760e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_a5ace0cb3df2538fb498.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_a5ace0cb3df2538fb498.png new file mode 100644 index 0000000000000000000000000000000000000000..c930334a938bfafcc5a9275d7e740ec5781c102c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_a5ace0cb3df2538fb498.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_bd3dcb8ce785525d5487.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_bd3dcb8ce785525d5487.png new file mode 100644 index 0000000000000000000000000000000000000000..e06104272aee08c12554829acd71dba35bc618b6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_bd3dcb8ce785525d5487.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_c09b2e9e85aa46b447a1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_c09b2e9e85aa46b447a1.png new file mode 100644 index 0000000000000000000000000000000000000000..a0e29a396e4b172b4c406d3bc00728a40d76751c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1432_c09b2e9e85aa46b447a1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_36f9aa3dad42c2857b92.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_36f9aa3dad42c2857b92.png new file mode 100644 index 0000000000000000000000000000000000000000..d6690b3d58b0778d7cde75ba10ed0738ce3dcaf0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_36f9aa3dad42c2857b92.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_c3be6299a9293bc4d02e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_c3be6299a9293bc4d02e.png new file mode 100644 index 0000000000000000000000000000000000000000..265bb17d79650619810b4b4adf2956391804eb4d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_c3be6299a9293bc4d02e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_e6f44d029048b1c1f306.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_e6f44d029048b1c1f306.png new file mode 100644 index 0000000000000000000000000000000000000000..e43def685d92fad6eea9ff3fd896fb11d88359c9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_e6f44d029048b1c1f306.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_f4b00c37b355d60cde54.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_f4b00c37b355d60cde54.png new file mode 100644 index 0000000000000000000000000000000000000000..32c762e52ef339a73f06e67472763ace0cd485b4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1440_f4b00c37b355d60cde54.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_268b0859bbaca6723e6a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_268b0859bbaca6723e6a.png new file mode 100644 index 0000000000000000000000000000000000000000..37a96ec7b71a046df469336993a331e75be77532 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_268b0859bbaca6723e6a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_4314f1796f83f2dc2195.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_4314f1796f83f2dc2195.png new file mode 100644 index 0000000000000000000000000000000000000000..648e0680939eccf4db1724f10ca4246c94b671c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_4314f1796f83f2dc2195.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_74b0779cd28b2472d148.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_74b0779cd28b2472d148.png new file mode 100644 index 0000000000000000000000000000000000000000..4f5dfbe2c23701da401640e0e8bca66b1c9aa450 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_74b0779cd28b2472d148.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_fa3457d0fce8bba4f078.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_fa3457d0fce8bba4f078.png new file mode 100644 index 0000000000000000000000000000000000000000..4c4d78cdce09608dc8e8a6b55b3ad4394a1a7b49 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1448_fa3457d0fce8bba4f078.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_21da72f5de55ac2c601b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_21da72f5de55ac2c601b.png new file mode 100644 index 0000000000000000000000000000000000000000..5275a6c46ab4984173663030924ece123310ed63 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_21da72f5de55ac2c601b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_2a98bd32e667f5d3ffa0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_2a98bd32e667f5d3ffa0.png new file mode 100644 index 0000000000000000000000000000000000000000..5ba77f2a6366e15f124b2a6c4eaf967d54d1000d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_2a98bd32e667f5d3ffa0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_9571275e76adeef75d31.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_9571275e76adeef75d31.png new file mode 100644 index 0000000000000000000000000000000000000000..971d268210c891ec9da72ae0f6120ffbcc604762 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_9571275e76adeef75d31.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_bf1d4866e27ceb3b9a7c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_bf1d4866e27ceb3b9a7c.png new file mode 100644 index 0000000000000000000000000000000000000000..456153c2efe53fbb0bde484918211a20edeff1ec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_144_bf1d4866e27ceb3b9a7c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_1ea2e8f00cd9d7016b4c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_1ea2e8f00cd9d7016b4c.png new file mode 100644 index 0000000000000000000000000000000000000000..6cfb0d1f6aefe61e8bf36a377c44abbc9a485810 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_1ea2e8f00cd9d7016b4c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_2cc115b8258f18162f1a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_2cc115b8258f18162f1a.png new file mode 100644 index 0000000000000000000000000000000000000000..a7789348dffb3ea3f493656288154d7a7e86c186 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_2cc115b8258f18162f1a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_79e985401ee07adaf747.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_79e985401ee07adaf747.png new file mode 100644 index 0000000000000000000000000000000000000000..71cbbf7c0c0d63cf726bc9ac23f283d06b04da76 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_79e985401ee07adaf747.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_d34a21e6b56035864f15.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_d34a21e6b56035864f15.png new file mode 100644 index 0000000000000000000000000000000000000000..f7878fd87cc380cbd922847aced076ad808dbd5e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1456_d34a21e6b56035864f15.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_0ca40d1889e8c981f83b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_0ca40d1889e8c981f83b.png new file mode 100644 index 0000000000000000000000000000000000000000..e7418caa4c15e5a4bffdbff5788df10fa433ad43 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_0ca40d1889e8c981f83b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_19080724f5041cc8877a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_19080724f5041cc8877a.png new file mode 100644 index 0000000000000000000000000000000000000000..1be76fc4177ba897418278ba8e38cba54aea5c05 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_19080724f5041cc8877a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_3e948f3d4a9690e3e832.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_3e948f3d4a9690e3e832.png new file mode 100644 index 0000000000000000000000000000000000000000..d14b42c9d195dc36f3860ca77b6780ca5e5ef473 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_3e948f3d4a9690e3e832.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_ef2794b66f8354aeb131.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_ef2794b66f8354aeb131.png new file mode 100644 index 0000000000000000000000000000000000000000..b08825cd569005fe05b39c3e0a3ca58c83454e87 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1464_ef2794b66f8354aeb131.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_97171e5e266bd9576202.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_97171e5e266bd9576202.png new file mode 100644 index 0000000000000000000000000000000000000000..4259ca645e4d80d1951d622395da3eccea6f9d8a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_97171e5e266bd9576202.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_e88bd6f6b5daeddb588f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_e88bd6f6b5daeddb588f.png new file mode 100644 index 0000000000000000000000000000000000000000..ca706f3bb11b6631eb50506c1ad2529ec67472b3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_e88bd6f6b5daeddb588f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_ec9e711d039b6031a67b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_ec9e711d039b6031a67b.png new file mode 100644 index 0000000000000000000000000000000000000000..659f1fb27d061933563487227d28385d2132c591 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_ec9e711d039b6031a67b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_f3df87bdd58ea75008e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_f3df87bdd58ea75008e9.png new file mode 100644 index 0000000000000000000000000000000000000000..bc04139a2b45f44a195a8e3f5515e256369fc768 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1472_f3df87bdd58ea75008e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_03ac52985174dcc37f37.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_03ac52985174dcc37f37.png new file mode 100644 index 0000000000000000000000000000000000000000..1727b8c9b3a8ffdbc09f98ef03ccfed70923517b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_03ac52985174dcc37f37.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_389503b866d34e567119.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_389503b866d34e567119.png new file mode 100644 index 0000000000000000000000000000000000000000..980732156f4ee2a8d524097f8c7b20b0c5545901 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_389503b866d34e567119.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_b5501bb70ba4c344c2a7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_b5501bb70ba4c344c2a7.png new file mode 100644 index 0000000000000000000000000000000000000000..f99af76f82403aaa37ab55983301efe8146666bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_b5501bb70ba4c344c2a7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_c13f5759d561a664b48e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_c13f5759d561a664b48e.png new file mode 100644 index 0000000000000000000000000000000000000000..103660486e262dbbd1aff00cc77d5f8c162f7a95 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1480_c13f5759d561a664b48e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_7a327cfc0f0d14000d66.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_7a327cfc0f0d14000d66.png new file mode 100644 index 0000000000000000000000000000000000000000..8a017eee51dda12e7e4089691264d33503498662 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_7a327cfc0f0d14000d66.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_945435b42cdef1f698ac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_945435b42cdef1f698ac.png new file mode 100644 index 0000000000000000000000000000000000000000..c927ddb64d9018b84b2c69412b4623d6a556beb9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_945435b42cdef1f698ac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_bad17279d6c78d388689.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_bad17279d6c78d388689.png new file mode 100644 index 0000000000000000000000000000000000000000..36e81606ebd5255fcd3d6ec7619b454a52db5340 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_bad17279d6c78d388689.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_d8f7bbf4bace619f94c8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_d8f7bbf4bace619f94c8.png new file mode 100644 index 0000000000000000000000000000000000000000..e45ab762c5c18217620d8fc318e6c3d8ef86cf98 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1488_d8f7bbf4bace619f94c8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_5999e1ed8ed0b37baa88.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_5999e1ed8ed0b37baa88.png new file mode 100644 index 0000000000000000000000000000000000000000..d2a0780fa79ada4bd0339a04f23529a128d5355f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_5999e1ed8ed0b37baa88.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_792b9ac80f3c91021509.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_792b9ac80f3c91021509.png new file mode 100644 index 0000000000000000000000000000000000000000..e2f7f224f3c7e2160e8f2a5e734ec34070a5a454 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_792b9ac80f3c91021509.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_861caf343730e243e96c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_861caf343730e243e96c.png new file mode 100644 index 0000000000000000000000000000000000000000..ba23cdf4a0fe5b1589578f3316127d647452f493 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_861caf343730e243e96c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_f3cbc6942eee44231d8a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_f3cbc6942eee44231d8a.png new file mode 100644 index 0000000000000000000000000000000000000000..a2bee76ad1276184a02ea0fd310f67f22f81d1b1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1496_f3cbc6942eee44231d8a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_1629e05d318559a66a72.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_1629e05d318559a66a72.png new file mode 100644 index 0000000000000000000000000000000000000000..a857f4eda227cae339ea62a588f917c4541bf142 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_1629e05d318559a66a72.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_53154f693252da98516a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_53154f693252da98516a.png new file mode 100644 index 0000000000000000000000000000000000000000..ade37284c4f6c408c2aadbd4b45368296d21dc17 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_53154f693252da98516a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_b30643dac647697eb3e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_b30643dac647697eb3e3.png new file mode 100644 index 0000000000000000000000000000000000000000..7eb90dd8c1baa08d534129b2d911380550e53364 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_b30643dac647697eb3e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_fb85857165e5c2b83d35.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_fb85857165e5c2b83d35.png new file mode 100644 index 0000000000000000000000000000000000000000..f071ed8613b6c99a2c0c9a6403fa67e4b8fcd6f3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1504_fb85857165e5c2b83d35.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_76d0a37e65b519005e16.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_76d0a37e65b519005e16.png new file mode 100644 index 0000000000000000000000000000000000000000..3da7d502b1ff0b26045817383805b6034d157de0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_76d0a37e65b519005e16.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_9628924d5a8c1579e38e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_9628924d5a8c1579e38e.png new file mode 100644 index 0000000000000000000000000000000000000000..95120d8006719683dab8a0c0f0acd7a03e0a68c0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_9628924d5a8c1579e38e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_ac8f638669bb1938fe1e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_ac8f638669bb1938fe1e.png new file mode 100644 index 0000000000000000000000000000000000000000..e69363c1ae8322329062cb422fa3a183e9b1f4ad Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_ac8f638669bb1938fe1e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_dc90df180ea054ba86a6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_dc90df180ea054ba86a6.png new file mode 100644 index 0000000000000000000000000000000000000000..298fb0eefa8bfdcba60b971b64377185290296c6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1512_dc90df180ea054ba86a6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_4acf2882ee3fdf5b3a10.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_4acf2882ee3fdf5b3a10.png new file mode 100644 index 0000000000000000000000000000000000000000..dc15cac352a8758e4f430d0c0c814129b4e495f0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_4acf2882ee3fdf5b3a10.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_759fd5f584726cd89a08.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_759fd5f584726cd89a08.png new file mode 100644 index 0000000000000000000000000000000000000000..5cc494d8296b3d225bd8d80eb1d9afccc89306bd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_759fd5f584726cd89a08.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_cbf2c6d45cd68fb8ce0d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_cbf2c6d45cd68fb8ce0d.png new file mode 100644 index 0000000000000000000000000000000000000000..1d18dc5f7033c2342db719d07202cfc0d1c7d3ea Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_cbf2c6d45cd68fb8ce0d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_da08c97000f72b3ea979.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_da08c97000f72b3ea979.png new file mode 100644 index 0000000000000000000000000000000000000000..270dbcb1118cc043d7381fcbf0f7e3798b5d51b1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1520_da08c97000f72b3ea979.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_175b3055df42cae41485.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_175b3055df42cae41485.png new file mode 100644 index 0000000000000000000000000000000000000000..237c9eb910c78522068bd1a17056d39fb7fc2b7c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_175b3055df42cae41485.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_27dd1112c1ee345cd5a7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_27dd1112c1ee345cd5a7.png new file mode 100644 index 0000000000000000000000000000000000000000..2d8b4a51e22255016a4e37a6f21c9ae2a05f5d02 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_27dd1112c1ee345cd5a7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_2e0f5eda4c827c82d01f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_2e0f5eda4c827c82d01f.png new file mode 100644 index 0000000000000000000000000000000000000000..ccbf1aa1559578636cb944cb6327eee1e2546d9d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_2e0f5eda4c827c82d01f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_823ba070d344cad76878.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_823ba070d344cad76878.png new file mode 100644 index 0000000000000000000000000000000000000000..1e68e7bb54aef4289f24c8ec2974e4eab39fcc02 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1528_823ba070d344cad76878.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_630cc9c3ad8164670f80.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_630cc9c3ad8164670f80.png new file mode 100644 index 0000000000000000000000000000000000000000..054cfed3934fd42c6c0cb4a5975e96879faa2eb6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_630cc9c3ad8164670f80.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_e20fdb183cf6dd559eb8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_e20fdb183cf6dd559eb8.png new file mode 100644 index 0000000000000000000000000000000000000000..8a0a50052fe30e2bf580d514bd493828db948fc3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_e20fdb183cf6dd559eb8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_e43b38eafcadeda16c8c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_e43b38eafcadeda16c8c.png new file mode 100644 index 0000000000000000000000000000000000000000..8382d3d0563c90fc7a436b5e22a55ddd7da3b53a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_e43b38eafcadeda16c8c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_eb8b309f2e7d0c0900fe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_eb8b309f2e7d0c0900fe.png new file mode 100644 index 0000000000000000000000000000000000000000..c5bd944ccc5b2d5fb0f45506ee4a220e721ab73c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_152_eb8b309f2e7d0c0900fe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_08a161065bf6a4856204.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_08a161065bf6a4856204.png new file mode 100644 index 0000000000000000000000000000000000000000..c092f8be0b9c973049d2fdf549272d381e9a9870 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_08a161065bf6a4856204.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_3fc0a6e0fd102232715a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_3fc0a6e0fd102232715a.png new file mode 100644 index 0000000000000000000000000000000000000000..ed049957905c6b66f90ebae88384b3ba6afec2f6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_3fc0a6e0fd102232715a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_71bad97180edd0fb5ddd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_71bad97180edd0fb5ddd.png new file mode 100644 index 0000000000000000000000000000000000000000..9ec856a2c560b15e33a31bcba3d8ff8157188208 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_71bad97180edd0fb5ddd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_d45ac8dd546b5c439b82.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_d45ac8dd546b5c439b82.png new file mode 100644 index 0000000000000000000000000000000000000000..3a5bb1f8b840dfaf1057d73b27d9825a19ec7d13 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1536_d45ac8dd546b5c439b82.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_3c1a45e0401e280d0a21.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_3c1a45e0401e280d0a21.png new file mode 100644 index 0000000000000000000000000000000000000000..3b1e5a71ed7ad14c62f05f372486cad133e976df Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_3c1a45e0401e280d0a21.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_bf2932c9751754389a5c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_bf2932c9751754389a5c.png new file mode 100644 index 0000000000000000000000000000000000000000..d2a1ad59c744131f96df020f30b49a8d430cbc33 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_bf2932c9751754389a5c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_de785629b33cb381b74b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_de785629b33cb381b74b.png new file mode 100644 index 0000000000000000000000000000000000000000..86e5e0b10198eb7006f070777f2cede9d9e90afa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_de785629b33cb381b74b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_f2506e5ffc80bfd166ad.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_f2506e5ffc80bfd166ad.png new file mode 100644 index 0000000000000000000000000000000000000000..826186c880cd4de549991b4c6154a973ed7acde0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1544_f2506e5ffc80bfd166ad.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_75bb106df0919e3df204.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_75bb106df0919e3df204.png new file mode 100644 index 0000000000000000000000000000000000000000..6293dd52911c38bd3e676e28543162f36c56a1fc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_75bb106df0919e3df204.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_8d1e832ac023537c101c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_8d1e832ac023537c101c.png new file mode 100644 index 0000000000000000000000000000000000000000..8cfdfcc1aaa3ecdbd1a14824e23a67e5e767ee3b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_8d1e832ac023537c101c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_aa4cc079d3d9191a8770.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_aa4cc079d3d9191a8770.png new file mode 100644 index 0000000000000000000000000000000000000000..ae9c8a55b018811df427b2d51cec4fbfeea23c1e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_aa4cc079d3d9191a8770.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_b723e293e89f77fe239f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_b723e293e89f77fe239f.png new file mode 100644 index 0000000000000000000000000000000000000000..e97bffac3d648e11008bcf59325d8adf32626f86 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1552_b723e293e89f77fe239f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_080450d9e4c3f1b32250.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_080450d9e4c3f1b32250.png new file mode 100644 index 0000000000000000000000000000000000000000..8f42b7d6b0793350aa57e34f1ef6abc3e7eb283f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_080450d9e4c3f1b32250.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_7ff55320d57bf225ae68.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_7ff55320d57bf225ae68.png new file mode 100644 index 0000000000000000000000000000000000000000..1a3d7450f889b1beff9fb6dd51f462ff6c7163df Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_7ff55320d57bf225ae68.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_92d3ec020ffb72174fba.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_92d3ec020ffb72174fba.png new file mode 100644 index 0000000000000000000000000000000000000000..326aa625f86d89169f6b848e6c6849194c2df5c9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_92d3ec020ffb72174fba.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_c97c1d61c8724e14c00b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_c97c1d61c8724e14c00b.png new file mode 100644 index 0000000000000000000000000000000000000000..2cc1a3951c9298cdcdb7abdb496720375ac10d64 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1560_c97c1d61c8724e14c00b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_45dc21f5d3d178d54339.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_45dc21f5d3d178d54339.png new file mode 100644 index 0000000000000000000000000000000000000000..b8b663bc1a74fab33201a6d42a98002e1ba4663e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_45dc21f5d3d178d54339.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_67b6d1876f24f6e8c3e4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_67b6d1876f24f6e8c3e4.png new file mode 100644 index 0000000000000000000000000000000000000000..06fa6efb23e08d781c7a37d72cdc3bafd0b5c1ca Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_67b6d1876f24f6e8c3e4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_a041609e0e3c44c0a850.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_a041609e0e3c44c0a850.png new file mode 100644 index 0000000000000000000000000000000000000000..b668a10657ea6f37111bd1bc6eb0f61c5e61c44a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_a041609e0e3c44c0a850.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_b803421655b7dc310cf8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_b803421655b7dc310cf8.png new file mode 100644 index 0000000000000000000000000000000000000000..63309d4ea70a68a4c4a856e937aa00cb46216ffe Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1568_b803421655b7dc310cf8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_2104b767ef4f17684879.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_2104b767ef4f17684879.png new file mode 100644 index 0000000000000000000000000000000000000000..c45206b86d869c75dd8d70bb67e4b9eebf9e8eb9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_2104b767ef4f17684879.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_4f1e1d5fc289cbc9fa89.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_4f1e1d5fc289cbc9fa89.png new file mode 100644 index 0000000000000000000000000000000000000000..5f26657676cb23c11829235266577f4d6fcf2e89 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_4f1e1d5fc289cbc9fa89.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_a69d5aa10180a8853110.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_a69d5aa10180a8853110.png new file mode 100644 index 0000000000000000000000000000000000000000..6a5214dd47b0d22880c986ceda7878e239ec62a7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_a69d5aa10180a8853110.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_c1fc87dd1314b4adf472.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_c1fc87dd1314b4adf472.png new file mode 100644 index 0000000000000000000000000000000000000000..ba538f1b4babf0e6dad498dc74a039573958de22 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1576_c1fc87dd1314b4adf472.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_4047a2d99cc326c5a57b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_4047a2d99cc326c5a57b.png new file mode 100644 index 0000000000000000000000000000000000000000..7295960ecff377c0509e609fdab8114c9d1ea520 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_4047a2d99cc326c5a57b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_5060670cca612a4e440f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_5060670cca612a4e440f.png new file mode 100644 index 0000000000000000000000000000000000000000..4464cd40defa5201651dcaaa0d32a1133c6ffb12 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_5060670cca612a4e440f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_6d080ebd872af391cc12.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_6d080ebd872af391cc12.png new file mode 100644 index 0000000000000000000000000000000000000000..3221fa4935bce83a66a11565f599927d5825f46b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_6d080ebd872af391cc12.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_fc7f4c75961dd33ba081.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_fc7f4c75961dd33ba081.png new file mode 100644 index 0000000000000000000000000000000000000000..09e1d6ec1b65ad682026bf7c9adf429589547ce2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1584_fc7f4c75961dd33ba081.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_00402f5cbaef44c29659.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_00402f5cbaef44c29659.png new file mode 100644 index 0000000000000000000000000000000000000000..663b5325969405d1783f74a9d9e76f6cd5e1a38e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_00402f5cbaef44c29659.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_92ac374f16616a3f52d5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_92ac374f16616a3f52d5.png new file mode 100644 index 0000000000000000000000000000000000000000..7186572b16e5560c4862360c876ff637a275ad98 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_92ac374f16616a3f52d5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_97278c134a95be731717.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_97278c134a95be731717.png new file mode 100644 index 0000000000000000000000000000000000000000..23d5c3572422eb057653a831398b092e047e01ee Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_97278c134a95be731717.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_f48579df1612af3504f2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_f48579df1612af3504f2.png new file mode 100644 index 0000000000000000000000000000000000000000..ce5fbd3db829f5ee1704509b4a0569852d926b79 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1592_f48579df1612af3504f2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_0f6bf7c1ef77ecb39e6c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_0f6bf7c1ef77ecb39e6c.png new file mode 100644 index 0000000000000000000000000000000000000000..43f93cc58f9ec9bc0a01a1b17b856fc8c19a71f0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_0f6bf7c1ef77ecb39e6c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_1759c1e65a73f7f13991.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_1759c1e65a73f7f13991.png new file mode 100644 index 0000000000000000000000000000000000000000..85f3ce37047e5b09501fb1b0656a63cc55327bd5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_1759c1e65a73f7f13991.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_699d0df245df08e0830f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_699d0df245df08e0830f.png new file mode 100644 index 0000000000000000000000000000000000000000..b4ebe90ac35dca710b9f7c9f37136451053f600b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_699d0df245df08e0830f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_deb755fc1e7c3773dac4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_deb755fc1e7c3773dac4.png new file mode 100644 index 0000000000000000000000000000000000000000..bfe94be828ecf53c47c40d8d6c3830a7cb0fadf2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1600_deb755fc1e7c3773dac4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_47f23ae6bc7899d2f737.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_47f23ae6bc7899d2f737.png new file mode 100644 index 0000000000000000000000000000000000000000..0d565eaedcc2f2fe68ac75271e88791e1c6bbc76 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_47f23ae6bc7899d2f737.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_6d2196a2c690a306717d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_6d2196a2c690a306717d.png new file mode 100644 index 0000000000000000000000000000000000000000..343b54b6d645ebca837a71d5989b59e329d560f7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_6d2196a2c690a306717d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_7ae5a05f8fc5b0d59741.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_7ae5a05f8fc5b0d59741.png new file mode 100644 index 0000000000000000000000000000000000000000..6b16e3c64de158aa6f0a696be2647b7d2bd79a41 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_7ae5a05f8fc5b0d59741.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_da4f2d17fb83c3abf79b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_da4f2d17fb83c3abf79b.png new file mode 100644 index 0000000000000000000000000000000000000000..db6f238b571d0578b28a5a09fe2d6ff3634283cd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1608_da4f2d17fb83c3abf79b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_268e9a732b517d634b0f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_268e9a732b517d634b0f.png new file mode 100644 index 0000000000000000000000000000000000000000..6c71226a78f04dca94336072235eb4a0dee973d3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_268e9a732b517d634b0f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_e8dd26e2169153e2c665.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_e8dd26e2169153e2c665.png new file mode 100644 index 0000000000000000000000000000000000000000..79a0bb6d6d0baa08d7a3840c0979f4019d1126d6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_e8dd26e2169153e2c665.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_f9cc3e65bbd82b475a59.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_f9cc3e65bbd82b475a59.png new file mode 100644 index 0000000000000000000000000000000000000000..c63a7954b944e86a7072502291f09ef8b212a185 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_f9cc3e65bbd82b475a59.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_fd584a28b09e2fa88b66.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_fd584a28b09e2fa88b66.png new file mode 100644 index 0000000000000000000000000000000000000000..dc33c9b1bec9e192f6a512bf7758a1d5f5710b5d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_160_fd584a28b09e2fa88b66.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_08b89fa693ea9f058799.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_08b89fa693ea9f058799.png new file mode 100644 index 0000000000000000000000000000000000000000..050494d418a2c88d6308f3592ec26e5303e284a6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_08b89fa693ea9f058799.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_23f6c16a47fdbf946082.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_23f6c16a47fdbf946082.png new file mode 100644 index 0000000000000000000000000000000000000000..f8a7e3845ac66f40163e9fba2d4f97fb843d0a11 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_23f6c16a47fdbf946082.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_4162b0a1073cede299c3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_4162b0a1073cede299c3.png new file mode 100644 index 0000000000000000000000000000000000000000..3e066db49aab1b9573160011d213617f8e37d0a9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_4162b0a1073cede299c3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_5cc14ce0bb3ca15eff14.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_5cc14ce0bb3ca15eff14.png new file mode 100644 index 0000000000000000000000000000000000000000..f9e882b72059d5fb8530f601299d0d8418f0a0f4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1616_5cc14ce0bb3ca15eff14.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_2d21b8cf80456a450416.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_2d21b8cf80456a450416.png new file mode 100644 index 0000000000000000000000000000000000000000..5419e90164d7ff426c47e158cf7380b4b33aeee3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_2d21b8cf80456a450416.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_87fd4f364cf3c1a5d743.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_87fd4f364cf3c1a5d743.png new file mode 100644 index 0000000000000000000000000000000000000000..a2cc9c4cafc7c4dc99ef75adfb4b05ee33beee23 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_87fd4f364cf3c1a5d743.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_c5c2b67c71ebaa893e3d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_c5c2b67c71ebaa893e3d.png new file mode 100644 index 0000000000000000000000000000000000000000..43cbd25b56b6c915a0f8ab47ed03a1d6e70ae8bc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_c5c2b67c71ebaa893e3d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_f2a4008163e3b0de1c67.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_f2a4008163e3b0de1c67.png new file mode 100644 index 0000000000000000000000000000000000000000..ae140e0bd1bca6f8d38919a497138e143cebfe80 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1624_f2a4008163e3b0de1c67.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_147308a3bc862cc2ef12.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_147308a3bc862cc2ef12.png new file mode 100644 index 0000000000000000000000000000000000000000..f59196209f68e260297ce861949f7b74dc347a8e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_147308a3bc862cc2ef12.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_4f352d796eb262af149d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_4f352d796eb262af149d.png new file mode 100644 index 0000000000000000000000000000000000000000..6a2abcad76a43557d3e3f25cd5b504330b207682 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_4f352d796eb262af149d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_87c3258c1b9a48214036.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_87c3258c1b9a48214036.png new file mode 100644 index 0000000000000000000000000000000000000000..80c25b22d7f75fc844ddb56ba7cb39eac17ca54d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_87c3258c1b9a48214036.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_ea4f6f4c802c31284d78.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_ea4f6f4c802c31284d78.png new file mode 100644 index 0000000000000000000000000000000000000000..0f6b11a78d1acbd8b72dffa169cb52f79471aea2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1632_ea4f6f4c802c31284d78.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_0dd87108f2fea6cda5da.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_0dd87108f2fea6cda5da.png new file mode 100644 index 0000000000000000000000000000000000000000..1e5404fc93a9f6a67785cfdee8000ef969169bf6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_0dd87108f2fea6cda5da.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_134a552e8977908b132a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_134a552e8977908b132a.png new file mode 100644 index 0000000000000000000000000000000000000000..9d37100149e5a82e1a01890fb61db643a26b6015 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_134a552e8977908b132a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_3d4aeb5bd69764009277.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_3d4aeb5bd69764009277.png new file mode 100644 index 0000000000000000000000000000000000000000..e383a9220642ccc558150e8848d2766eea2645cf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_3d4aeb5bd69764009277.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_41d647d9e8df50b6d062.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_41d647d9e8df50b6d062.png new file mode 100644 index 0000000000000000000000000000000000000000..dd4c04e0100eec1215bac10723937b3aa5f649a9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1640_41d647d9e8df50b6d062.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_1de3e9f61db08c1fa525.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_1de3e9f61db08c1fa525.png new file mode 100644 index 0000000000000000000000000000000000000000..ff78d2cea55457d27eb036ec3f5128aef77e58ca Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_1de3e9f61db08c1fa525.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_3801cc005fba469011fe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_3801cc005fba469011fe.png new file mode 100644 index 0000000000000000000000000000000000000000..9ba4e196bd1396947ed3b85736f75ffa2351be69 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_3801cc005fba469011fe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_ae897ffe1c6081601953.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_ae897ffe1c6081601953.png new file mode 100644 index 0000000000000000000000000000000000000000..75773c95271070357a8d23ed099dbaed22f266e3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_ae897ffe1c6081601953.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_cab21230e1e43c4dd2f2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_cab21230e1e43c4dd2f2.png new file mode 100644 index 0000000000000000000000000000000000000000..7fe3cc720a724b71b5302f325f16807ff6fa25de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1648_cab21230e1e43c4dd2f2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_55996a575df11d966c0a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_55996a575df11d966c0a.png new file mode 100644 index 0000000000000000000000000000000000000000..2e402befa70b7e0b4199075720664ce7c43e4f36 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_55996a575df11d966c0a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_8c3aa89e0dc3c0cfd217.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_8c3aa89e0dc3c0cfd217.png new file mode 100644 index 0000000000000000000000000000000000000000..798ab8253ea20d38422011d6b998c53a326433c7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_8c3aa89e0dc3c0cfd217.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_e4d7864777608c38c24e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_e4d7864777608c38c24e.png new file mode 100644 index 0000000000000000000000000000000000000000..847427f3e7e9bd71a837c8cd57568b435a29cb58 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_e4d7864777608c38c24e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_e5c0e9634965058135d3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_e5c0e9634965058135d3.png new file mode 100644 index 0000000000000000000000000000000000000000..b61cffc4f6111cf6040877520e2c26e130344427 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1656_e5c0e9634965058135d3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_2ae67e741780fdc97d63.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_2ae67e741780fdc97d63.png new file mode 100644 index 0000000000000000000000000000000000000000..dd595a247aecf37ff7fc48cf5e2f21ff7d320678 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_2ae67e741780fdc97d63.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_66dbe96487807e61c639.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_66dbe96487807e61c639.png new file mode 100644 index 0000000000000000000000000000000000000000..a9f445a55e6159fba2e5080a295352626e18a324 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_66dbe96487807e61c639.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_97cb7ab131b642c3b7bc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_97cb7ab131b642c3b7bc.png new file mode 100644 index 0000000000000000000000000000000000000000..3a3e6dac19e7e1f523c248e6ac1a386ba519d0d5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_97cb7ab131b642c3b7bc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_ef66c85637afc9c7e435.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_ef66c85637afc9c7e435.png new file mode 100644 index 0000000000000000000000000000000000000000..408a940a6cc3a3512a5f9178cecb1215d704460a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1664_ef66c85637afc9c7e435.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_02de7ad6917c63f45a9a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_02de7ad6917c63f45a9a.png new file mode 100644 index 0000000000000000000000000000000000000000..5438f07ff241adf94d0b6a18f9d3ced13ac352b7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_02de7ad6917c63f45a9a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_32cff90f0713a31af381.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_32cff90f0713a31af381.png new file mode 100644 index 0000000000000000000000000000000000000000..c69e92403aeab04e25b2a14bd00629547f9da3cf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_32cff90f0713a31af381.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_379652fb1beafd6c8e09.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_379652fb1beafd6c8e09.png new file mode 100644 index 0000000000000000000000000000000000000000..9ef5ae0c51f83b759b3ded144b573f8ec7c525fd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_379652fb1beafd6c8e09.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_f950a91f8f5d756c5764.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_f950a91f8f5d756c5764.png new file mode 100644 index 0000000000000000000000000000000000000000..78fce7a2fd986b4767725eab35bce2c5f43f46e3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1672_f950a91f8f5d756c5764.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_3015d5a2cea98cfefb90.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_3015d5a2cea98cfefb90.png new file mode 100644 index 0000000000000000000000000000000000000000..0181c0140499d054c0092a01ba7c61a92d0dc1d9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_3015d5a2cea98cfefb90.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_34a40e5cb3d7a0052b50.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_34a40e5cb3d7a0052b50.png new file mode 100644 index 0000000000000000000000000000000000000000..21493d82e3d11b09fb8e20e6557ca30c7088b914 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_34a40e5cb3d7a0052b50.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_3d0c633b59f3f6e54d50.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_3d0c633b59f3f6e54d50.png new file mode 100644 index 0000000000000000000000000000000000000000..a56da47b6863318b584795b7e38f15ad3e2d8c18 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_3d0c633b59f3f6e54d50.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_e0d685807d82b1fe3333.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_e0d685807d82b1fe3333.png new file mode 100644 index 0000000000000000000000000000000000000000..6d788aac34dba01958c5080ac2476d9a790f34bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1680_e0d685807d82b1fe3333.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_47be1430323289df7f7b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_47be1430323289df7f7b.png new file mode 100644 index 0000000000000000000000000000000000000000..9c5ab761a3ddf531983a660943848aada565a037 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_47be1430323289df7f7b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_59c336d7ac9a01cc96ed.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_59c336d7ac9a01cc96ed.png new file mode 100644 index 0000000000000000000000000000000000000000..696d7abdd7082e8bb389157ea57c234cba8da97e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_59c336d7ac9a01cc96ed.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_78b53a833655e5e7ddfd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_78b53a833655e5e7ddfd.png new file mode 100644 index 0000000000000000000000000000000000000000..43926763be6e37486ab36898f2a862d035289c30 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_78b53a833655e5e7ddfd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_afc0af5c54864aff9b73.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_afc0af5c54864aff9b73.png new file mode 100644 index 0000000000000000000000000000000000000000..4908f63b9f79e61b9d3b68d81674534054730ec2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1688_afc0af5c54864aff9b73.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_01bbee240219c25af928.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_01bbee240219c25af928.png new file mode 100644 index 0000000000000000000000000000000000000000..f5af98d36070154ec47eb930b3ddf847858ea263 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_01bbee240219c25af928.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_770dcafceef728f19c85.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_770dcafceef728f19c85.png new file mode 100644 index 0000000000000000000000000000000000000000..01321a521789a625710d6b75f949e9090f82c402 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_770dcafceef728f19c85.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_7a6f1b9b97ae1a405c3d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_7a6f1b9b97ae1a405c3d.png new file mode 100644 index 0000000000000000000000000000000000000000..d29f8976b2b9354e332c2f6901c8d071571db9de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_7a6f1b9b97ae1a405c3d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_fe40b1e9c3e598d742f2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_fe40b1e9c3e598d742f2.png new file mode 100644 index 0000000000000000000000000000000000000000..d35abaaab6c542eefc0f04a2942292e19e3b654d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_168_fe40b1e9c3e598d742f2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_50897643f723faf8a1d8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_50897643f723faf8a1d8.png new file mode 100644 index 0000000000000000000000000000000000000000..a633f35aa2c92057e5c01c161a750bd75b841eb7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_50897643f723faf8a1d8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_ac0d4f4b95333e8b92f1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_ac0d4f4b95333e8b92f1.png new file mode 100644 index 0000000000000000000000000000000000000000..dfadecc8ce4145d6ba561fd8dd94dd5562d126aa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_ac0d4f4b95333e8b92f1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_d73227af624327a56f89.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_d73227af624327a56f89.png new file mode 100644 index 0000000000000000000000000000000000000000..a50a3440afcdd061fb1347775b554a0ed0a3c2ba Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_d73227af624327a56f89.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_e4fbe946ca939c5f2907.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_e4fbe946ca939c5f2907.png new file mode 100644 index 0000000000000000000000000000000000000000..056a31108a679d6b74bcdc018d99a3af19130830 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1696_e4fbe946ca939c5f2907.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_0cf3546cb22517953a42.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_0cf3546cb22517953a42.png new file mode 100644 index 0000000000000000000000000000000000000000..fb804a21f1385c5ad01c1d776f19d9e1859222ba Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_0cf3546cb22517953a42.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_53e698e35096606e0dae.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_53e698e35096606e0dae.png new file mode 100644 index 0000000000000000000000000000000000000000..781b5eaddaf034d2b454ba3e4a4f1dc6ec6a0109 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_53e698e35096606e0dae.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_ba6f49bf4d359df02f2b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_ba6f49bf4d359df02f2b.png new file mode 100644 index 0000000000000000000000000000000000000000..c36399922b40af32c3fd31b5bf4baf0fbfa18f80 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_ba6f49bf4d359df02f2b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_e935b3df8a409038c0e2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_e935b3df8a409038c0e2.png new file mode 100644 index 0000000000000000000000000000000000000000..971181fb319e245c2816b8ab555d7882151bc339 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_16_e935b3df8a409038c0e2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_314bebb79cb782edfd8f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_314bebb79cb782edfd8f.png new file mode 100644 index 0000000000000000000000000000000000000000..56b6ce404bb0d65a4f611dd50302d0a2918381d0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_314bebb79cb782edfd8f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_42a268d779b7667ccc2e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_42a268d779b7667ccc2e.png new file mode 100644 index 0000000000000000000000000000000000000000..af61ca290d12634f28755eb01a15b740cdcde27f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_42a268d779b7667ccc2e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_e0d8a90f9602209761f6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_e0d8a90f9602209761f6.png new file mode 100644 index 0000000000000000000000000000000000000000..21a2012bb3bca873f40e9e9d320fc9247ce2b8b1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_e0d8a90f9602209761f6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_e1dcbbc29cdb423960a2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_e1dcbbc29cdb423960a2.png new file mode 100644 index 0000000000000000000000000000000000000000..e3e5c6fa3eae9516db8c83269d834d94ac268bc3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1704_e1dcbbc29cdb423960a2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_27d02e0fde32d79961e2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_27d02e0fde32d79961e2.png new file mode 100644 index 0000000000000000000000000000000000000000..b2892dab255d9b2d23167c2d692f7519bb157e50 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_27d02e0fde32d79961e2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_50df1a6368b684b0eabd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_50df1a6368b684b0eabd.png new file mode 100644 index 0000000000000000000000000000000000000000..f6649669082e111362eb8ed32f4cdc8b2b394878 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_50df1a6368b684b0eabd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_b2393282cb27ff5aaf28.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_b2393282cb27ff5aaf28.png new file mode 100644 index 0000000000000000000000000000000000000000..b326475db46a3c2b0c450c572b7f69cac19663b5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_b2393282cb27ff5aaf28.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_bb78706228e3f64446a2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_bb78706228e3f64446a2.png new file mode 100644 index 0000000000000000000000000000000000000000..25462602ae807f3e308e49fb1c6a037bf178e82a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1712_bb78706228e3f64446a2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_5a01ec75660827b76fe7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_5a01ec75660827b76fe7.png new file mode 100644 index 0000000000000000000000000000000000000000..f6f71cd19e5a350add9c8150d6bef1e1d2519e08 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_5a01ec75660827b76fe7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_679437e5e8fa85b1b9fa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_679437e5e8fa85b1b9fa.png new file mode 100644 index 0000000000000000000000000000000000000000..644a541d00e797bea6d377fbc2391e6fab48acd2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_679437e5e8fa85b1b9fa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_973ea290aaf93bd8b3d2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_973ea290aaf93bd8b3d2.png new file mode 100644 index 0000000000000000000000000000000000000000..eae3ef4f992b714dee5e8bbf0670000b28671126 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_973ea290aaf93bd8b3d2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_c173e21a79da12ee1883.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_c173e21a79da12ee1883.png new file mode 100644 index 0000000000000000000000000000000000000000..c50a6930c37ce517062eca7da454b7afb5fffdf2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1720_c173e21a79da12ee1883.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_590c097e334a7ac52f79.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_590c097e334a7ac52f79.png new file mode 100644 index 0000000000000000000000000000000000000000..cdc415b8282ef65aee9ad8a3e00be0edf13fc392 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_590c097e334a7ac52f79.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_61df962c93eb4afff8f0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_61df962c93eb4afff8f0.png new file mode 100644 index 0000000000000000000000000000000000000000..9058d6174474ed0790206a8a9a3cfe790163e4f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_61df962c93eb4afff8f0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_89f458f901461e553af6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_89f458f901461e553af6.png new file mode 100644 index 0000000000000000000000000000000000000000..a41a868ecb9d2663c99dd323a7509c50d3ee9903 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_89f458f901461e553af6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_f006b6af0fba27faf76e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_f006b6af0fba27faf76e.png new file mode 100644 index 0000000000000000000000000000000000000000..d9a250c56c958bf06eb4f3a735d6a307d8303fd8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1728_f006b6af0fba27faf76e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_1029fe384454cadd60b1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_1029fe384454cadd60b1.png new file mode 100644 index 0000000000000000000000000000000000000000..186284876d5e669782495da3f7760ca863e03943 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_1029fe384454cadd60b1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_23c66688cd6e5a6be908.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_23c66688cd6e5a6be908.png new file mode 100644 index 0000000000000000000000000000000000000000..caa011110cc0c0e9eac60f5240f132e62c2e8228 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_23c66688cd6e5a6be908.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_81a3ccf88a1ed0a25cef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_81a3ccf88a1ed0a25cef.png new file mode 100644 index 0000000000000000000000000000000000000000..8148882404b0927a5e53529239182cb4fc412cd2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_81a3ccf88a1ed0a25cef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_9f5d69e25ec444d8827e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_9f5d69e25ec444d8827e.png new file mode 100644 index 0000000000000000000000000000000000000000..cb8a426651a5b3b03cd9f051d10877aab0971f3b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1736_9f5d69e25ec444d8827e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_5a12380a065557e23f86.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_5a12380a065557e23f86.png new file mode 100644 index 0000000000000000000000000000000000000000..d710312d1c56a28f40864e8e1283144feb3db3e0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_5a12380a065557e23f86.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_9f5c0597dd918b1fd1cd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_9f5c0597dd918b1fd1cd.png new file mode 100644 index 0000000000000000000000000000000000000000..17563c8c4913ac6ccf68317ee27446368e4b4647 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_9f5c0597dd918b1fd1cd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_ce9d3bc192f9ffe8e69b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_ce9d3bc192f9ffe8e69b.png new file mode 100644 index 0000000000000000000000000000000000000000..05d118b45595156680d1a7565aee4c29a0aeab5e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_ce9d3bc192f9ffe8e69b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_d7f69c8955653068af1c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_d7f69c8955653068af1c.png new file mode 100644 index 0000000000000000000000000000000000000000..4a0489ad084892373554a5e326b3abd0819fbcec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1744_d7f69c8955653068af1c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_1fca70034e43e1efd16b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_1fca70034e43e1efd16b.png new file mode 100644 index 0000000000000000000000000000000000000000..e475ca24151abcd50cab56aa1d3e258d405626e3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_1fca70034e43e1efd16b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_2d2c4c4207dded6ac310.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_2d2c4c4207dded6ac310.png new file mode 100644 index 0000000000000000000000000000000000000000..2bd9a14523d1813587ab1cfb891802b5b5e65ec2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_2d2c4c4207dded6ac310.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_41340b7ef5a230ba2c1d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_41340b7ef5a230ba2c1d.png new file mode 100644 index 0000000000000000000000000000000000000000..a8d26f456f168f72d9a31e7f7165c2cdd6bbb70a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_41340b7ef5a230ba2c1d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_c0be450d736d315a7f7f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_c0be450d736d315a7f7f.png new file mode 100644 index 0000000000000000000000000000000000000000..4921d90db2433424363e2b0803ce493b447df974 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1752_c0be450d736d315a7f7f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_1a916468c5f244822b28.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_1a916468c5f244822b28.png new file mode 100644 index 0000000000000000000000000000000000000000..4311273d0d085d6742d8b915f639c40ed5b1b8b0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_1a916468c5f244822b28.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_2b0a2a64666aeb10c8d5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_2b0a2a64666aeb10c8d5.png new file mode 100644 index 0000000000000000000000000000000000000000..e3a7b29bae78873f0bfbaed0f714589bec488f4f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_2b0a2a64666aeb10c8d5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_963ce470d918d6ff96d4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_963ce470d918d6ff96d4.png new file mode 100644 index 0000000000000000000000000000000000000000..7ecd4731d318f3ec3bad56e1da3d9f80095e4446 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_963ce470d918d6ff96d4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_f9e33a1e2eb7c8bf304f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_f9e33a1e2eb7c8bf304f.png new file mode 100644 index 0000000000000000000000000000000000000000..1db4dbf80407490b7399c6a45efe4a1a54fc4847 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1760_f9e33a1e2eb7c8bf304f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_2dae9d2ec2b54e62fa2a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_2dae9d2ec2b54e62fa2a.png new file mode 100644 index 0000000000000000000000000000000000000000..589b5819c368dd67dd6a10b72022c053f62f4508 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_2dae9d2ec2b54e62fa2a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_bad2b0cfefb3891be2e4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_bad2b0cfefb3891be2e4.png new file mode 100644 index 0000000000000000000000000000000000000000..1055edff196e1c266cf773aec868abed0da1191a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_bad2b0cfefb3891be2e4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_bfde8140b1fc4f9db23e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_bfde8140b1fc4f9db23e.png new file mode 100644 index 0000000000000000000000000000000000000000..138b1d04efb4fb05daa0572d1ad3ec2e9ce40a8d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_bfde8140b1fc4f9db23e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_cfcc80a8ccc0659ac78e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_cfcc80a8ccc0659ac78e.png new file mode 100644 index 0000000000000000000000000000000000000000..34254ef0b9aef8ec839b3645932e203f17444343 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1768_cfcc80a8ccc0659ac78e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_77496af6afd4fcfc7802.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_77496af6afd4fcfc7802.png new file mode 100644 index 0000000000000000000000000000000000000000..aa7f2aeb71167fae327a37361d6e26fcd3e64cec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_77496af6afd4fcfc7802.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_7c3251a3bc6842a14eaf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_7c3251a3bc6842a14eaf.png new file mode 100644 index 0000000000000000000000000000000000000000..902a12f14796f0e0763081534861ddbf702c0159 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_7c3251a3bc6842a14eaf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_b57338304a381b55774f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_b57338304a381b55774f.png new file mode 100644 index 0000000000000000000000000000000000000000..65dd684df00603728f53c89414d9ea8bea8d07bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_b57338304a381b55774f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_ddb9188a2ecb3ba1a967.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_ddb9188a2ecb3ba1a967.png new file mode 100644 index 0000000000000000000000000000000000000000..c55ce6bbc09411d3c1b6d530e1ea360721c1a3b7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_176_ddb9188a2ecb3ba1a967.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_145e5db4ddf1294e74ca.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_145e5db4ddf1294e74ca.png new file mode 100644 index 0000000000000000000000000000000000000000..eadfdec8bb73edfe53405abaf72095c36c4ba973 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_145e5db4ddf1294e74ca.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_1dfeb68dada29a861d6d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_1dfeb68dada29a861d6d.png new file mode 100644 index 0000000000000000000000000000000000000000..fd654e1405486ffe7d951c22d3f3cce3cf57dd57 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_1dfeb68dada29a861d6d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_7c1cf0a21ce762d708cc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_7c1cf0a21ce762d708cc.png new file mode 100644 index 0000000000000000000000000000000000000000..96320ce808964c2fc57a96cf5b6e63ad3b2796ac Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_7c1cf0a21ce762d708cc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_a6253244e0fcd16b6309.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_a6253244e0fcd16b6309.png new file mode 100644 index 0000000000000000000000000000000000000000..944c9caa8cc17aef0eb3dfc6d1adc256675ecaa6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1776_a6253244e0fcd16b6309.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_0b7130c5c030a08cc234.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_0b7130c5c030a08cc234.png new file mode 100644 index 0000000000000000000000000000000000000000..c50493cdd082ffc6564bc87a90bc5c235e7d20db Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_0b7130c5c030a08cc234.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_2f7a93c89a8e73332a71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_2f7a93c89a8e73332a71.png new file mode 100644 index 0000000000000000000000000000000000000000..2ca098be8174569c1586800ee19b0d4b01c3c689 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_2f7a93c89a8e73332a71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_40c1334312f0eaa4a409.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_40c1334312f0eaa4a409.png new file mode 100644 index 0000000000000000000000000000000000000000..1d7ec15c270e8db0d6e958232a73cdc826c2d4cd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_40c1334312f0eaa4a409.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_7e32e1185f32b992ee09.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_7e32e1185f32b992ee09.png new file mode 100644 index 0000000000000000000000000000000000000000..13d972e8e517336e325b0e924b95402c058a6b30 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1784_7e32e1185f32b992ee09.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_03a513699d32547cfcce.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_03a513699d32547cfcce.png new file mode 100644 index 0000000000000000000000000000000000000000..b762354509c3c7d15de1cd7b59544ea8791be96f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_03a513699d32547cfcce.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_5119d4d7a2257dd71e9b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_5119d4d7a2257dd71e9b.png new file mode 100644 index 0000000000000000000000000000000000000000..dadb03be3a82d438629d51ade5168e6c75a9e25a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_5119d4d7a2257dd71e9b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_6c8c3c85979fc0112886.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_6c8c3c85979fc0112886.png new file mode 100644 index 0000000000000000000000000000000000000000..e1c45d948193129d8c1345f8e49566ff6c03e186 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_6c8c3c85979fc0112886.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_e03fe724b53ff007f509.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_e03fe724b53ff007f509.png new file mode 100644 index 0000000000000000000000000000000000000000..04164fc80902a8b0a71ca582c3f0f813a639cf01 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1792_e03fe724b53ff007f509.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_1d80862a2e619ee149af.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_1d80862a2e619ee149af.png new file mode 100644 index 0000000000000000000000000000000000000000..ea1707661a3e14102b5e06b818030ae9cd43377d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_1d80862a2e619ee149af.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_29829117e355f84867a9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_29829117e355f84867a9.png new file mode 100644 index 0000000000000000000000000000000000000000..84696a2be5c32224b26b8184162112a211a27fe8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_29829117e355f84867a9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_595d783a674b31607abe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_595d783a674b31607abe.png new file mode 100644 index 0000000000000000000000000000000000000000..d512f122cca71f1bcef7eab7de367a7fe77901d9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_595d783a674b31607abe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_ba06bc64108c60c3d249.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_ba06bc64108c60c3d249.png new file mode 100644 index 0000000000000000000000000000000000000000..bdb6e85d8dc782892468e1d757d5503b9433c7da Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1800_ba06bc64108c60c3d249.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_2c08b8408a4ed78c3242.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_2c08b8408a4ed78c3242.png new file mode 100644 index 0000000000000000000000000000000000000000..60697e91f76b8d5dc33cc96e0a8be068122e54af Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_2c08b8408a4ed78c3242.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_4e784e29fddbb72791ca.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_4e784e29fddbb72791ca.png new file mode 100644 index 0000000000000000000000000000000000000000..81b36ad29b6023209153624ab61c85da2a264760 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_4e784e29fddbb72791ca.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_60edf5874ce5e54fff8a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_60edf5874ce5e54fff8a.png new file mode 100644 index 0000000000000000000000000000000000000000..a44aa82b86c185042e0df4aaab7be4a15e24f6bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_60edf5874ce5e54fff8a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_e42f3a5cff465c0326eb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_e42f3a5cff465c0326eb.png new file mode 100644 index 0000000000000000000000000000000000000000..739761d080e32d2a0d2a39bd65cb86308d638d15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1808_e42f3a5cff465c0326eb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_0b84c7e55854de6abb35.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_0b84c7e55854de6abb35.png new file mode 100644 index 0000000000000000000000000000000000000000..9b8443d7d8090b1bafd0e01d25507b8b7e1157ee Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_0b84c7e55854de6abb35.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_6ce5b89f865c878ca7ce.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_6ce5b89f865c878ca7ce.png new file mode 100644 index 0000000000000000000000000000000000000000..d89816dfd3b98ec9240483d53f0e5f7be13d1807 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_6ce5b89f865c878ca7ce.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_c0afacd8e639ac88c950.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_c0afacd8e639ac88c950.png new file mode 100644 index 0000000000000000000000000000000000000000..c53a25284fd93106a321a212c67b90134c473620 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_c0afacd8e639ac88c950.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_db34427fe5dc5f158ded.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_db34427fe5dc5f158ded.png new file mode 100644 index 0000000000000000000000000000000000000000..77db0f3315c86d4c302850b453421bd93a10a72b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1816_db34427fe5dc5f158ded.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_2018a3ead09df8a1c987.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_2018a3ead09df8a1c987.png new file mode 100644 index 0000000000000000000000000000000000000000..c899c4495a52dc672d6a0e1c62f4505d40cb71be Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_2018a3ead09df8a1c987.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_9cf90d8af441b007efe6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_9cf90d8af441b007efe6.png new file mode 100644 index 0000000000000000000000000000000000000000..b915e63c0979ca7d425daa026e480a47d4f31ab1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_9cf90d8af441b007efe6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_de127351776d646ede0d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_de127351776d646ede0d.png new file mode 100644 index 0000000000000000000000000000000000000000..d09d7f5af3eb314a49a7c79a8deb115f67e1db8f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_de127351776d646ede0d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_ffbfc753df3dec5c3208.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_ffbfc753df3dec5c3208.png new file mode 100644 index 0000000000000000000000000000000000000000..7aec0c8ff4437d6e58533899c4dd530345a8e27e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1824_ffbfc753df3dec5c3208.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_1bfea247631943876000.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_1bfea247631943876000.png new file mode 100644 index 0000000000000000000000000000000000000000..6dd9c96b90891ea0ce4542a8573aa898f22fc7b8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_1bfea247631943876000.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_5e23f48d1d42652a0b10.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_5e23f48d1d42652a0b10.png new file mode 100644 index 0000000000000000000000000000000000000000..72c57391496a191b572b60ee14b929b8271b56aa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_5e23f48d1d42652a0b10.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_7a8f27b94cbeac050b37.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_7a8f27b94cbeac050b37.png new file mode 100644 index 0000000000000000000000000000000000000000..210b1610086b6c7e4cad277eef538cd32b6fe303 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_7a8f27b94cbeac050b37.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_9cd399d776df586c9c02.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_9cd399d776df586c9c02.png new file mode 100644 index 0000000000000000000000000000000000000000..36bff6c216dafaa26ea9d9ee1c753bdde4ef248a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1832_9cd399d776df586c9c02.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_2e537d6ae5539024016c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_2e537d6ae5539024016c.png new file mode 100644 index 0000000000000000000000000000000000000000..9a173c9829e80a575156f356d2ae4fad679a0591 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_2e537d6ae5539024016c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_4fb89ef66c812201003f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_4fb89ef66c812201003f.png new file mode 100644 index 0000000000000000000000000000000000000000..bd4f58d8cf4ac230f86334e8401e830954dcac71 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_4fb89ef66c812201003f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_beb7c5efe82f32f2c89f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_beb7c5efe82f32f2c89f.png new file mode 100644 index 0000000000000000000000000000000000000000..112005eeede16725c172a40e555f4e49f9690dce Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_beb7c5efe82f32f2c89f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_eacff8e0f39e8da08765.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_eacff8e0f39e8da08765.png new file mode 100644 index 0000000000000000000000000000000000000000..4b92f7cbecbfe61e56307b1dfee7a6e97f8de841 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1840_eacff8e0f39e8da08765.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_08204171975de6aa75a9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_08204171975de6aa75a9.png new file mode 100644 index 0000000000000000000000000000000000000000..fb6c7e5de270d7e1ad3d8018f35fe9d443b16a75 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_08204171975de6aa75a9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_1be533c9b8642ccbd69e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_1be533c9b8642ccbd69e.png new file mode 100644 index 0000000000000000000000000000000000000000..f5b9e6380b7fa28b64bbe007a26a748a9009f39a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_1be533c9b8642ccbd69e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_98bfc990a3604dddcca6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_98bfc990a3604dddcca6.png new file mode 100644 index 0000000000000000000000000000000000000000..c90120c00107f73b40f42acd8d07f2b9544a6d5b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_98bfc990a3604dddcca6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_d9322c759e78020c14f9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_d9322c759e78020c14f9.png new file mode 100644 index 0000000000000000000000000000000000000000..6601297dacfc1a79d73908ca5b13a434c67dbdc6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1848_d9322c759e78020c14f9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_7c9b69b4f13b5c0abece.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_7c9b69b4f13b5c0abece.png new file mode 100644 index 0000000000000000000000000000000000000000..f89268be0e3d89a748e4c0a80faf66444506f259 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_7c9b69b4f13b5c0abece.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_973f6a3c3ee07eda891b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_973f6a3c3ee07eda891b.png new file mode 100644 index 0000000000000000000000000000000000000000..5042910f80bd61f18194bdbb5533fc45826491af Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_973f6a3c3ee07eda891b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_b07a08bc9ab86b402e2a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_b07a08bc9ab86b402e2a.png new file mode 100644 index 0000000000000000000000000000000000000000..713e0db8a06b0ddd3a6b64779ada85aece51244c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_b07a08bc9ab86b402e2a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_e6bdd88d9dc7c75d4ca3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_e6bdd88d9dc7c75d4ca3.png new file mode 100644 index 0000000000000000000000000000000000000000..8e68a4e67e9f13ba7c52c52718d253cbf9bca7b7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_184_e6bdd88d9dc7c75d4ca3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_27b928ab82c5575c233c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_27b928ab82c5575c233c.png new file mode 100644 index 0000000000000000000000000000000000000000..a5824d491eb6e4bf72b241553c106142f7223b1a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_27b928ab82c5575c233c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_9f028c76839d73eab2ab.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_9f028c76839d73eab2ab.png new file mode 100644 index 0000000000000000000000000000000000000000..19af0cb082b32eef163365ed999f51f99384e922 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_9f028c76839d73eab2ab.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_acd7add95c524a43efc7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_acd7add95c524a43efc7.png new file mode 100644 index 0000000000000000000000000000000000000000..19f77aad558f8b1dc0492a7e8fb80433158b88dd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_acd7add95c524a43efc7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_fc9e1244d8b7441727e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_fc9e1244d8b7441727e9.png new file mode 100644 index 0000000000000000000000000000000000000000..788904e24783a9762608f2f7ddd9149e203dcee7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1856_fc9e1244d8b7441727e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_2cb8e9d19416aa855ae3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_2cb8e9d19416aa855ae3.png new file mode 100644 index 0000000000000000000000000000000000000000..ab66bb2337e222683d851ea9908f833681b5edf9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_2cb8e9d19416aa855ae3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_3af6cca038957854954d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_3af6cca038957854954d.png new file mode 100644 index 0000000000000000000000000000000000000000..a2c1a4bc216782d1c53997b7043c4ebe265591bd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_3af6cca038957854954d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_bb9b59dfd7940d706691.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_bb9b59dfd7940d706691.png new file mode 100644 index 0000000000000000000000000000000000000000..a0af447c57246e5e685060a433c77bd84e4cdb54 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_bb9b59dfd7940d706691.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_ec496ac094d6a7c53ab6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_ec496ac094d6a7c53ab6.png new file mode 100644 index 0000000000000000000000000000000000000000..aa8094d329dd0d601cbde57f4da69849d0c7fdfe Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1864_ec496ac094d6a7c53ab6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_31e589344f5ef8978c9f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_31e589344f5ef8978c9f.png new file mode 100644 index 0000000000000000000000000000000000000000..1588280520fc0b213fd1f102a34370cf4fa7dd34 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_31e589344f5ef8978c9f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_65231e5433fd842a9973.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_65231e5433fd842a9973.png new file mode 100644 index 0000000000000000000000000000000000000000..75f3822ab8bff5cfc4f3972fec629afdcab4a813 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_65231e5433fd842a9973.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_c36006f7e46b670362b4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_c36006f7e46b670362b4.png new file mode 100644 index 0000000000000000000000000000000000000000..2060c219987f75a1ac69529826b0b5b294ddb900 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_c36006f7e46b670362b4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_cdeb27bcabdc8cb919a1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_cdeb27bcabdc8cb919a1.png new file mode 100644 index 0000000000000000000000000000000000000000..f1c8ef887419c083be0b584d604c3bbcc1cc960d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1872_cdeb27bcabdc8cb919a1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_16f97c54584e8c176782.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_16f97c54584e8c176782.png new file mode 100644 index 0000000000000000000000000000000000000000..a5789bdee158829fbc3aa04e0d9f170d4c4dc476 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_16f97c54584e8c176782.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_59266af13de06f4cd472.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_59266af13de06f4cd472.png new file mode 100644 index 0000000000000000000000000000000000000000..d500af6afbd6397e936c8a59670092be932ebf1e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_59266af13de06f4cd472.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_75325ad11ccd36e3bca1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_75325ad11ccd36e3bca1.png new file mode 100644 index 0000000000000000000000000000000000000000..cfa3cf4558a3761ce70c2da002c7ef51a384ec00 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_75325ad11ccd36e3bca1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_c848d9be75adb5960650.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_c848d9be75adb5960650.png new file mode 100644 index 0000000000000000000000000000000000000000..270b1d1d867eb1b6ad587b802b6feeb36b8df46c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1880_c848d9be75adb5960650.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_3adb0b07a95c158b84ad.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_3adb0b07a95c158b84ad.png new file mode 100644 index 0000000000000000000000000000000000000000..8ec2a16c78750657d45f4564e52dc346fa6c3f91 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_3adb0b07a95c158b84ad.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_7b75911ccdb678b3a976.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_7b75911ccdb678b3a976.png new file mode 100644 index 0000000000000000000000000000000000000000..866f7730a3b3830cfd562bc070b0b84fe253c837 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_7b75911ccdb678b3a976.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_86d54203ecfaca6291d0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_86d54203ecfaca6291d0.png new file mode 100644 index 0000000000000000000000000000000000000000..8c58977d342b97c7224bdc6077f081663ef5b9bc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_86d54203ecfaca6291d0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_8ae9cfb1af9ccf7c1a1d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_8ae9cfb1af9ccf7c1a1d.png new file mode 100644 index 0000000000000000000000000000000000000000..fa3bc3b98891d9c7f40e78ad9483a038c415d2c3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1888_8ae9cfb1af9ccf7c1a1d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_27c5036394cf299adb47.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_27c5036394cf299adb47.png new file mode 100644 index 0000000000000000000000000000000000000000..7fb6d0cf71f2a3924e78c42b4318552d019f848c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_27c5036394cf299adb47.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_29f4e682c32068be6ee6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_29f4e682c32068be6ee6.png new file mode 100644 index 0000000000000000000000000000000000000000..0efda5151a3ce2f870f79a7d69329b31cfad4bee Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_29f4e682c32068be6ee6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_43076fbb48e6ed29a61c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_43076fbb48e6ed29a61c.png new file mode 100644 index 0000000000000000000000000000000000000000..6e82e7136caecc14fb40e8dfecf2ff08f405d379 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_43076fbb48e6ed29a61c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_a000f6436ad45ec3f551.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_a000f6436ad45ec3f551.png new file mode 100644 index 0000000000000000000000000000000000000000..4e4e0b58f05297e9989ebf602f1cb158f884cf36 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1896_a000f6436ad45ec3f551.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_234baf3f279d560d92d2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_234baf3f279d560d92d2.png new file mode 100644 index 0000000000000000000000000000000000000000..8caa038e7aa548ce74262ca74a7880f3c53ee393 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_234baf3f279d560d92d2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_35be072a700b769038db.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_35be072a700b769038db.png new file mode 100644 index 0000000000000000000000000000000000000000..de191dc642012ef5e977bb83cc831cc966d6e395 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_35be072a700b769038db.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_a8928482b1db701915ac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_a8928482b1db701915ac.png new file mode 100644 index 0000000000000000000000000000000000000000..c633cfe2080983ed64d7d381875e2024013d2979 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_a8928482b1db701915ac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_c1886dcd749cbc60adfd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_c1886dcd749cbc60adfd.png new file mode 100644 index 0000000000000000000000000000000000000000..c60860d9e9b5a06d89bb8bd164d184ca583d8e09 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1904_c1886dcd749cbc60adfd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_3adb61482ec02a0583d5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_3adb61482ec02a0583d5.png new file mode 100644 index 0000000000000000000000000000000000000000..2ce3ed500bc31d9e766ae041bbf482f9599f81a1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_3adb61482ec02a0583d5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_3af03303490f4370396f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_3af03303490f4370396f.png new file mode 100644 index 0000000000000000000000000000000000000000..d91573f7cf1c471eba0d85fbf3c27a4e5f6c7b76 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_3af03303490f4370396f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_9da4544d95a30d67cb32.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_9da4544d95a30d67cb32.png new file mode 100644 index 0000000000000000000000000000000000000000..b64dca1a25bed4ceace5bbde7da5f6fb70c6ed9e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_9da4544d95a30d67cb32.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_cdb5377e850db598c266.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_cdb5377e850db598c266.png new file mode 100644 index 0000000000000000000000000000000000000000..2234e2c921a553a4afd19e9551370fe830fbe1e2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1912_cdb5377e850db598c266.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_0d10a7dbf7251bbada04.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_0d10a7dbf7251bbada04.png new file mode 100644 index 0000000000000000000000000000000000000000..19c7bd50d54181ebb8d700e059295701534d7f0e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_0d10a7dbf7251bbada04.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_6d4d45fd2048c6be252f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_6d4d45fd2048c6be252f.png new file mode 100644 index 0000000000000000000000000000000000000000..89d04a722297c4d9477d406b09264719b893b165 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_6d4d45fd2048c6be252f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_d289710871a92ba999e8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_d289710871a92ba999e8.png new file mode 100644 index 0000000000000000000000000000000000000000..c6ae2ec3b253ea9b2aaf8afe5d3f23a551f311e2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_d289710871a92ba999e8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_d5a4ff7de7a7153f2941.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_d5a4ff7de7a7153f2941.png new file mode 100644 index 0000000000000000000000000000000000000000..f0f2c82c3b8466e81ea4f71671be89dba0be6560 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1920_d5a4ff7de7a7153f2941.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_4bd865ae05d0074a2375.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_4bd865ae05d0074a2375.png new file mode 100644 index 0000000000000000000000000000000000000000..b8aa3840b49a781eba239e098869c4b0cd28bec5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_4bd865ae05d0074a2375.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_78729b8aaac1e6b08697.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_78729b8aaac1e6b08697.png new file mode 100644 index 0000000000000000000000000000000000000000..5389e924104de944c76f0bd8147b96d54b76770f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_78729b8aaac1e6b08697.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_a008fe9b5be650d7d99f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_a008fe9b5be650d7d99f.png new file mode 100644 index 0000000000000000000000000000000000000000..7d92c92a7d7f2a4efe705dcca425ba3b47634a9c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_a008fe9b5be650d7d99f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_af8fa5edbca9692cffc2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_af8fa5edbca9692cffc2.png new file mode 100644 index 0000000000000000000000000000000000000000..3e02ee79ca288cf082ceb34325fb8b0038e00cd1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1928_af8fa5edbca9692cffc2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_5fc6b92733f1b72454a9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_5fc6b92733f1b72454a9.png new file mode 100644 index 0000000000000000000000000000000000000000..169d9d36099acbbad51b6c63849d4d7453ac68de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_5fc6b92733f1b72454a9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_63bb647b973f8f813f9e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_63bb647b973f8f813f9e.png new file mode 100644 index 0000000000000000000000000000000000000000..e913cfd0dccfdfcbcf97568af32610c97ff390dc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_63bb647b973f8f813f9e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_9846167229bd54634f68.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_9846167229bd54634f68.png new file mode 100644 index 0000000000000000000000000000000000000000..b3ee2c2f9eac8814b7f09cc70e9dd06edfeb229a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_9846167229bd54634f68.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_d1e1f9c8aa7873e56eed.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_d1e1f9c8aa7873e56eed.png new file mode 100644 index 0000000000000000000000000000000000000000..78666b43705d7192d60c360333e6606898f4431d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_192_d1e1f9c8aa7873e56eed.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_35429031d120f2b96126.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_35429031d120f2b96126.png new file mode 100644 index 0000000000000000000000000000000000000000..5a5096e37a95605a8e09368bba4e2794f4f8027f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_35429031d120f2b96126.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_852acc36599591503eef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_852acc36599591503eef.png new file mode 100644 index 0000000000000000000000000000000000000000..6e0d259069c1a1a359805c1a268f3677f474e944 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_852acc36599591503eef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_dcec368a7622b3d8c0be.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_dcec368a7622b3d8c0be.png new file mode 100644 index 0000000000000000000000000000000000000000..cd9b1518b19751178ee083e037e9f77171ba3254 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_dcec368a7622b3d8c0be.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_f3a3fadee46766d2470a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_f3a3fadee46766d2470a.png new file mode 100644 index 0000000000000000000000000000000000000000..fe56d70fdf5f2d81d65f3523e0f36cf972f769a4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1936_f3a3fadee46766d2470a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_35adf9bd55486e1c564c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_35adf9bd55486e1c564c.png new file mode 100644 index 0000000000000000000000000000000000000000..9004afb495ef73b653b570d190a50da7b27c588e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_35adf9bd55486e1c564c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_78808717363c62ee0546.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_78808717363c62ee0546.png new file mode 100644 index 0000000000000000000000000000000000000000..5d5d66179dee62b56527a621e9c87a6850b86f12 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_78808717363c62ee0546.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_8177942062418e290d7d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_8177942062418e290d7d.png new file mode 100644 index 0000000000000000000000000000000000000000..01e4539086308ac6091ff1393af0062a121e8924 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_8177942062418e290d7d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_fa0c0398f5b77951fef8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_fa0c0398f5b77951fef8.png new file mode 100644 index 0000000000000000000000000000000000000000..05dc79b4e9d6700d9a1a9b72792c3b341d735445 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1944_fa0c0398f5b77951fef8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_01c4c69e0d5b531f80da.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_01c4c69e0d5b531f80da.png new file mode 100644 index 0000000000000000000000000000000000000000..0eba86c91d293a64c444eb31dc2b5b0295ab0aaa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_01c4c69e0d5b531f80da.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_6ad546e10ab053b15547.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_6ad546e10ab053b15547.png new file mode 100644 index 0000000000000000000000000000000000000000..9370beb974717619309cab0b14f243af6b6813e7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_6ad546e10ab053b15547.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_9eff640395fdb57cd9c7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_9eff640395fdb57cd9c7.png new file mode 100644 index 0000000000000000000000000000000000000000..52b00a498b80c321a857dde166432ae2c033d4eb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_9eff640395fdb57cd9c7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_b9e6d46bbf3ea0d7f03d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_b9e6d46bbf3ea0d7f03d.png new file mode 100644 index 0000000000000000000000000000000000000000..f9a527f5f238ceae6c93807b84e72d70b06bd9fc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1952_b9e6d46bbf3ea0d7f03d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_9263ee13eaef00dc177f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_9263ee13eaef00dc177f.png new file mode 100644 index 0000000000000000000000000000000000000000..8a8e8e90717b8bead50bfcd0a446e27121fc90e2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_9263ee13eaef00dc177f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_b35c1f83e65423b07bda.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_b35c1f83e65423b07bda.png new file mode 100644 index 0000000000000000000000000000000000000000..33cc13a0866652a2b803bddd7d91b701cb3ac10e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_b35c1f83e65423b07bda.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_d0a34bec2dd9fe1ba19d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_d0a34bec2dd9fe1ba19d.png new file mode 100644 index 0000000000000000000000000000000000000000..bda00c1c2bb931172358340e766f525c9ccf9f5b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_d0a34bec2dd9fe1ba19d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_f69ac92a12eeba479470.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_f69ac92a12eeba479470.png new file mode 100644 index 0000000000000000000000000000000000000000..9598f7f8a8d9cc9df2a7eee0a82c67bcbb4b03f0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1960_f69ac92a12eeba479470.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_212445186f3f2a7ed98e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_212445186f3f2a7ed98e.png new file mode 100644 index 0000000000000000000000000000000000000000..8db15b942110b13adb3a4c74f34c153218101f68 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_212445186f3f2a7ed98e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_b94497f03241b794f556.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_b94497f03241b794f556.png new file mode 100644 index 0000000000000000000000000000000000000000..e12ff475a544867ffd2910b2566daefa1ec39aa4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_b94497f03241b794f556.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_de75a859cd98957c62e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_de75a859cd98957c62e9.png new file mode 100644 index 0000000000000000000000000000000000000000..dd732e8fea3c1c9d3c545ef9ba5fa78102553025 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_de75a859cd98957c62e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_ff19fe998502f08ef355.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_ff19fe998502f08ef355.png new file mode 100644 index 0000000000000000000000000000000000000000..4957a8520b0baa7639951bfb394fed27ef9403bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1968_ff19fe998502f08ef355.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_0df481310c9adbf409ba.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_0df481310c9adbf409ba.png new file mode 100644 index 0000000000000000000000000000000000000000..1cd250434ff4c12b9985b8e547a5e21aefb37813 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_0df481310c9adbf409ba.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_2891304fc5901c3c1e71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_2891304fc5901c3c1e71.png new file mode 100644 index 0000000000000000000000000000000000000000..deb9612c4a337d0d1630d4396cfea1edf9ef5212 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_2891304fc5901c3c1e71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_ca58948c6f5e8d1e6c38.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_ca58948c6f5e8d1e6c38.png new file mode 100644 index 0000000000000000000000000000000000000000..ac6dcc08a2bbcfba8d052571bddb859511393426 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_ca58948c6f5e8d1e6c38.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_ec9a8a5c1f221d207e64.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_ec9a8a5c1f221d207e64.png new file mode 100644 index 0000000000000000000000000000000000000000..7f6eb119ec22f4e9df0007bc4d5fd7f3608d85c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1976_ec9a8a5c1f221d207e64.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_597e25f12127dac13b2a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_597e25f12127dac13b2a.png new file mode 100644 index 0000000000000000000000000000000000000000..cefeb16e6aa930da76c2d36dab64685651781265 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_597e25f12127dac13b2a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_61b9f23963df182bd12d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_61b9f23963df182bd12d.png new file mode 100644 index 0000000000000000000000000000000000000000..480126e625612272c28ee90d51247a300114cddb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_61b9f23963df182bd12d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_c4207687ea8ecd4f4a45.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_c4207687ea8ecd4f4a45.png new file mode 100644 index 0000000000000000000000000000000000000000..173b044b46cf6dc1b03d71fe331cfa6d9c0b44ad Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_c4207687ea8ecd4f4a45.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_cf4f7819de6a9d73ca74.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_cf4f7819de6a9d73ca74.png new file mode 100644 index 0000000000000000000000000000000000000000..473f0e30625305dfee1de7458c93c2ec502f8c9d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1984_cf4f7819de6a9d73ca74.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_281a32d10f5b0365fb14.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_281a32d10f5b0365fb14.png new file mode 100644 index 0000000000000000000000000000000000000000..e7cbd00e868b1a1c0869e825a5cc6b1a5a222e58 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_281a32d10f5b0365fb14.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_92aeaed1bdf499b11f79.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_92aeaed1bdf499b11f79.png new file mode 100644 index 0000000000000000000000000000000000000000..8b5f70bfb2ed1425364acee79a7f83456c2382d6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_92aeaed1bdf499b11f79.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_cf518b56981a6ad11247.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_cf518b56981a6ad11247.png new file mode 100644 index 0000000000000000000000000000000000000000..fe251ddff8a69c5a1ed00b755cd4fb9b87e3c538 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_cf518b56981a6ad11247.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_f8c5d5ec8577a44a670e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_f8c5d5ec8577a44a670e.png new file mode 100644 index 0000000000000000000000000000000000000000..701b4d929f975e8de236b9dca7474702dde08cb4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_1992_f8c5d5ec8577a44a670e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_11513f1b9a12e97321f9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_11513f1b9a12e97321f9.png new file mode 100644 index 0000000000000000000000000000000000000000..5bdf2f66728b2bd2eb12491a9fe99c7af287b5e1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_11513f1b9a12e97321f9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_4ee817ff55a50642bd59.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_4ee817ff55a50642bd59.png new file mode 100644 index 0000000000000000000000000000000000000000..ddc710d7f4513121aaebe0e34c37a4854d64267d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_4ee817ff55a50642bd59.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_56be1cccfe16e31d1eda.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_56be1cccfe16e31d1eda.png new file mode 100644 index 0000000000000000000000000000000000000000..d7f75b0953c2d66c46a8e084efa259ee3abab0e6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_56be1cccfe16e31d1eda.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_5747cd6b7dfc06af840d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_5747cd6b7dfc06af840d.png new file mode 100644 index 0000000000000000000000000000000000000000..2afc17dc38ddf59e1ef16f635f99f149c6bd0c20 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2000_5747cd6b7dfc06af840d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_67662513522c4b9f8fd5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_67662513522c4b9f8fd5.png new file mode 100644 index 0000000000000000000000000000000000000000..2305992ac20a60c409d87dc3964c360741412a7f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_67662513522c4b9f8fd5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_9c383de17af136bf0d80.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_9c383de17af136bf0d80.png new file mode 100644 index 0000000000000000000000000000000000000000..a9eb2534cbef286a64da3479e41c1e1843e39a98 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_9c383de17af136bf0d80.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_a0785c61ee79cf436c16.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_a0785c61ee79cf436c16.png new file mode 100644 index 0000000000000000000000000000000000000000..aa469407f5c8e47c97665750ca0ae4e58fc76308 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_a0785c61ee79cf436c16.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_edeeab747d59b9ede014.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_edeeab747d59b9ede014.png new file mode 100644 index 0000000000000000000000000000000000000000..c58565d9e9e4b12bec8761c7767af6106e26c66e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2008_edeeab747d59b9ede014.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_3dc0cb79df01ab7c84bc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_3dc0cb79df01ab7c84bc.png new file mode 100644 index 0000000000000000000000000000000000000000..1c3405147294581a5cc1768367a55196659e5802 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_3dc0cb79df01ab7c84bc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_7b675596c6407752ba34.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_7b675596c6407752ba34.png new file mode 100644 index 0000000000000000000000000000000000000000..9cb3012bf8dfb3d7b826eda2b6197a1af404385e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_7b675596c6407752ba34.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_96aa6aebfb42c1649c43.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_96aa6aebfb42c1649c43.png new file mode 100644 index 0000000000000000000000000000000000000000..1f125a2dea8480ac9b1d01d5b7870c23a979f5d1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_96aa6aebfb42c1649c43.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_b47039a85898acafffb2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_b47039a85898acafffb2.png new file mode 100644 index 0000000000000000000000000000000000000000..edbe265b59b813ac3c5287408d601ce94ae3ffa1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_200_b47039a85898acafffb2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_42552ac0dc72fef56307.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_42552ac0dc72fef56307.png new file mode 100644 index 0000000000000000000000000000000000000000..0b625029b47beb0d427619792d94ea8c62ffdcd5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_42552ac0dc72fef56307.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_7b8a29624a01f6e3b5aa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_7b8a29624a01f6e3b5aa.png new file mode 100644 index 0000000000000000000000000000000000000000..f145b3fa6ee8228ccbb85acf4550e42559c691b2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_7b8a29624a01f6e3b5aa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_b666077c67c4ed2ef995.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_b666077c67c4ed2ef995.png new file mode 100644 index 0000000000000000000000000000000000000000..312aa67ac9d66b5bbed4b710f662437a9304aaaf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_b666077c67c4ed2ef995.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_caa85d659cf2405bcefd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_caa85d659cf2405bcefd.png new file mode 100644 index 0000000000000000000000000000000000000000..92d82559ac29d014d5d74e6327a38a960eca53f7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2016_caa85d659cf2405bcefd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_39ba76f44c6320920da5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_39ba76f44c6320920da5.png new file mode 100644 index 0000000000000000000000000000000000000000..684245f0a84b7873bb592a48202681a6d6bf4a77 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_39ba76f44c6320920da5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_42449f3583b7d0ee5299.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_42449f3583b7d0ee5299.png new file mode 100644 index 0000000000000000000000000000000000000000..60202c94f8a478a45d7bbd181a47b2ce4ce8ffcb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_42449f3583b7d0ee5299.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_58e7ab91d438b3e02f2c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_58e7ab91d438b3e02f2c.png new file mode 100644 index 0000000000000000000000000000000000000000..14ac624c1da49338cd326a8be2931fa5a16bfce7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_58e7ab91d438b3e02f2c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_5cd6da1a61c49184c69c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_5cd6da1a61c49184c69c.png new file mode 100644 index 0000000000000000000000000000000000000000..d8afee2ee8a9d4bdeef295cd29a1e586283699b5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2024_5cd6da1a61c49184c69c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_0966289b7bf679cabff2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_0966289b7bf679cabff2.png new file mode 100644 index 0000000000000000000000000000000000000000..313cd537eb264360282460151d53a075fe33ccb6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_0966289b7bf679cabff2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_ab427828257bfc2f48cd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_ab427828257bfc2f48cd.png new file mode 100644 index 0000000000000000000000000000000000000000..8e4e202b7c1f3357526ccac3ebc0d374c5a18744 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_ab427828257bfc2f48cd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_e9dca02fa7653317e9d3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_e9dca02fa7653317e9d3.png new file mode 100644 index 0000000000000000000000000000000000000000..bb0017c05b6e0534aa0b701460503702766bec8d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_e9dca02fa7653317e9d3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_f92644e9ab4ae89453ab.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_f92644e9ab4ae89453ab.png new file mode 100644 index 0000000000000000000000000000000000000000..6b418471ef07ae95388619bf98fb368cb2caeece Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2032_f92644e9ab4ae89453ab.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_11ffc1e2966f22ed98b2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_11ffc1e2966f22ed98b2.png new file mode 100644 index 0000000000000000000000000000000000000000..9caf8f5a7b91023acbbf964ef5a8cfa09f3c07a0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_11ffc1e2966f22ed98b2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_1b5809dee200dda60230.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_1b5809dee200dda60230.png new file mode 100644 index 0000000000000000000000000000000000000000..6260b9228d572521cd196035f87740ece42346b7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_1b5809dee200dda60230.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_c4c3d69e91379eeb2517.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_c4c3d69e91379eeb2517.png new file mode 100644 index 0000000000000000000000000000000000000000..c49dc9cbb6b0d6cd367d03de94b59b49f11257b9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_c4c3d69e91379eeb2517.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_f6836c5b36ab58e6c0fb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_f6836c5b36ab58e6c0fb.png new file mode 100644 index 0000000000000000000000000000000000000000..44aadb3b3750ffc2263e1067445b6a737646597b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2040_f6836c5b36ab58e6c0fb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_123cf925aafba429d41f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_123cf925aafba429d41f.png new file mode 100644 index 0000000000000000000000000000000000000000..0f9ba14b705cf8e9c597feb7da929fa1f2da051e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_123cf925aafba429d41f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_2fac4432914c0d06816d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_2fac4432914c0d06816d.png new file mode 100644 index 0000000000000000000000000000000000000000..5268649f566b87db56b8f28b0457f20798328482 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_2fac4432914c0d06816d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_9132f798001f5c0ccc6d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_9132f798001f5c0ccc6d.png new file mode 100644 index 0000000000000000000000000000000000000000..18458b18f9fa3667d1b157f9e5d4063e5d705099 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_9132f798001f5c0ccc6d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_b964563afa4fd9125c79.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_b964563afa4fd9125c79.png new file mode 100644 index 0000000000000000000000000000000000000000..50774848002e78663c23333ff2387e43f04d69c3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2048_b964563afa4fd9125c79.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_119cb34b947beaab2613.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_119cb34b947beaab2613.png new file mode 100644 index 0000000000000000000000000000000000000000..04829c33d6ab80f99657d2adb84c63525ce85e91 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_119cb34b947beaab2613.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_ae649897728e92928855.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_ae649897728e92928855.png new file mode 100644 index 0000000000000000000000000000000000000000..fdfd614d93585ddba1d3652c54644adfb8cb57f6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_ae649897728e92928855.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_c30db4d32c0339da583c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_c30db4d32c0339da583c.png new file mode 100644 index 0000000000000000000000000000000000000000..e9f11755db32e12edba8d095d9d27b28776d7dfc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_c30db4d32c0339da583c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_f2d52946f625bac82075.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_f2d52946f625bac82075.png new file mode 100644 index 0000000000000000000000000000000000000000..a492026ea71c394533de87457c02f63e6f359c8f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2056_f2d52946f625bac82075.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_31a98ecd5357f46a64be.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_31a98ecd5357f46a64be.png new file mode 100644 index 0000000000000000000000000000000000000000..9602c7da0f0dd55c4544dfa45313430ab490663a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_31a98ecd5357f46a64be.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_519a4aa69108743c7ec6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_519a4aa69108743c7ec6.png new file mode 100644 index 0000000000000000000000000000000000000000..b9fb54e8c3bd8d179c9d1fbe225ba87ccf9d107a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_519a4aa69108743c7ec6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_b712e152cd356cd14c7a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_b712e152cd356cd14c7a.png new file mode 100644 index 0000000000000000000000000000000000000000..ac38e0d9eeef450461f2af63bcc4088f43adac9d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_b712e152cd356cd14c7a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_ea44735eac8f71037fce.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_ea44735eac8f71037fce.png new file mode 100644 index 0000000000000000000000000000000000000000..d384a72a3c6cc471a569fb29aaa7367f595e0639 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2064_ea44735eac8f71037fce.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_1bc4c894c72890f665a0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_1bc4c894c72890f665a0.png new file mode 100644 index 0000000000000000000000000000000000000000..473d2df242d2d6c268079b2ae91cc5a64e15dc64 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_1bc4c894c72890f665a0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_a7d9f847844b98f32c62.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_a7d9f847844b98f32c62.png new file mode 100644 index 0000000000000000000000000000000000000000..6f3e800b2b6aa8883a1fee2aca703e13f10f01d8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_a7d9f847844b98f32c62.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_ddac369b4ded67607317.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_ddac369b4ded67607317.png new file mode 100644 index 0000000000000000000000000000000000000000..2998c63a6e355473fa425ea1ef90fd8d95a64cb6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_ddac369b4ded67607317.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_e0f0cd526f1fbf142c07.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_e0f0cd526f1fbf142c07.png new file mode 100644 index 0000000000000000000000000000000000000000..91aa666a977fb50717599f1c011e3ee749c8737f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2072_e0f0cd526f1fbf142c07.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_234274db4799f23af905.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_234274db4799f23af905.png new file mode 100644 index 0000000000000000000000000000000000000000..df6ebb84f195ba3fc672a2c4671c55467a6f6efa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_234274db4799f23af905.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_470d734462cd775639c9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_470d734462cd775639c9.png new file mode 100644 index 0000000000000000000000000000000000000000..55baa9eefb0b32c6e0ab3eaf612258601acac9f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_470d734462cd775639c9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_b5332c71d8ad6af0f6cb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_b5332c71d8ad6af0f6cb.png new file mode 100644 index 0000000000000000000000000000000000000000..ef985c6b5241993a47c809b1a75fe74e70d05a95 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_b5332c71d8ad6af0f6cb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_c0c4ed778328664db5ef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_c0c4ed778328664db5ef.png new file mode 100644 index 0000000000000000000000000000000000000000..6c3b769854be070a424f6276f5188ccf70384b1f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2080_c0c4ed778328664db5ef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_1177c53f90c63a53fae4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_1177c53f90c63a53fae4.png new file mode 100644 index 0000000000000000000000000000000000000000..dbaa0f7524929fb0db5d35fca53b3452b91d1ed3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_1177c53f90c63a53fae4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_5726b36383a81db55440.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_5726b36383a81db55440.png new file mode 100644 index 0000000000000000000000000000000000000000..5e72c53bfe8021a8144ef9d7ff3b27d528c5fe9e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_5726b36383a81db55440.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_6d9f398496e3d9fb86ac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_6d9f398496e3d9fb86ac.png new file mode 100644 index 0000000000000000000000000000000000000000..04d64c6b4ca9f74eb5b9e0413aa68e58f7d6b801 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_6d9f398496e3d9fb86ac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_ad4894c9ed610fe1ef61.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_ad4894c9ed610fe1ef61.png new file mode 100644 index 0000000000000000000000000000000000000000..d3090b5830bc913413528bad32ee65167c07dc7a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2088_ad4894c9ed610fe1ef61.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_75c3a361aae943fc98c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_75c3a361aae943fc98c0.png new file mode 100644 index 0000000000000000000000000000000000000000..9d73753e801c82d5b010c41ecc5b4d79b495ce29 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_75c3a361aae943fc98c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_8f8d4515856693522ab8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_8f8d4515856693522ab8.png new file mode 100644 index 0000000000000000000000000000000000000000..fcc4d1ee0d729ceffc70b6e370073f001f2237f3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_8f8d4515856693522ab8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_b0925536eb78c8a0506f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_b0925536eb78c8a0506f.png new file mode 100644 index 0000000000000000000000000000000000000000..8df82a48c6fe76e632cd349dabe3b0b035eccdf5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_b0925536eb78c8a0506f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_b6b72350a5e90be236b8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_b6b72350a5e90be236b8.png new file mode 100644 index 0000000000000000000000000000000000000000..c01ee879668c5c9b20e77be0ac0b70aaa5cf51b0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_208_b6b72350a5e90be236b8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_3917a475eb07dd7d8014.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_3917a475eb07dd7d8014.png new file mode 100644 index 0000000000000000000000000000000000000000..23badd9adc4fbd011a3111ecb58480bed6517229 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_3917a475eb07dd7d8014.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_52636a9fab6f23c15f25.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_52636a9fab6f23c15f25.png new file mode 100644 index 0000000000000000000000000000000000000000..71f1e3696f9b8e767b62c3f1938624ee2c9c3fbc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_52636a9fab6f23c15f25.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_ab63da1d97e801140f79.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_ab63da1d97e801140f79.png new file mode 100644 index 0000000000000000000000000000000000000000..3a777d88fa6c511cd572b3004d567034f1a6517d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_ab63da1d97e801140f79.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_b3898442a9a4b0221ae6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_b3898442a9a4b0221ae6.png new file mode 100644 index 0000000000000000000000000000000000000000..5b7d8ef27e18bfef88d5ceeb92ed807417a9b631 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2096_b3898442a9a4b0221ae6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_17fbe5a0dfd4b8fd48f2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_17fbe5a0dfd4b8fd48f2.png new file mode 100644 index 0000000000000000000000000000000000000000..c99ab7f7900fdd861def4197333c216ad0fcf3c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_17fbe5a0dfd4b8fd48f2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_619ebebabd9789748cac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_619ebebabd9789748cac.png new file mode 100644 index 0000000000000000000000000000000000000000..07820250aba891dea98999f1ccaa4b9aed61d896 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_619ebebabd9789748cac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_8ff4357dcc31e396a35b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_8ff4357dcc31e396a35b.png new file mode 100644 index 0000000000000000000000000000000000000000..9f7bed132ab3aeaaf0c40e0630e4d80ee4198ea7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_8ff4357dcc31e396a35b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_f2ed3d7c9f2ca2721159.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_f2ed3d7c9f2ca2721159.png new file mode 100644 index 0000000000000000000000000000000000000000..0b26833b2f32622244d74f7a84102b0482a9c23a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2104_f2ed3d7c9f2ca2721159.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_99bcea6ecb4b26371b0a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_99bcea6ecb4b26371b0a.png new file mode 100644 index 0000000000000000000000000000000000000000..657a2c3b62d2ef41563ffc5abd0aafe19477b0a9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_99bcea6ecb4b26371b0a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_de58ff517f7ad93b7980.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_de58ff517f7ad93b7980.png new file mode 100644 index 0000000000000000000000000000000000000000..bb76985db607f2c0d42c15af97df077ba88372db Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_de58ff517f7ad93b7980.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_ee9b23aa9877d821342d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_ee9b23aa9877d821342d.png new file mode 100644 index 0000000000000000000000000000000000000000..1e93d188e6958b6ff27af432d3d7154f7f2d2a53 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_ee9b23aa9877d821342d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_fdc939f4f2cfbd53196d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_fdc939f4f2cfbd53196d.png new file mode 100644 index 0000000000000000000000000000000000000000..1e50fe038dd36f418afc7994895cbaf42ca81259 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2112_fdc939f4f2cfbd53196d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_0cc11534506dbd7f7bac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_0cc11534506dbd7f7bac.png new file mode 100644 index 0000000000000000000000000000000000000000..43db67eee9a4eb1d504956897496765358081a21 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_0cc11534506dbd7f7bac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_4dd49f0520bc4123cc57.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_4dd49f0520bc4123cc57.png new file mode 100644 index 0000000000000000000000000000000000000000..c638d227ef60b332fbaf88c56559fd3805552c1a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_4dd49f0520bc4123cc57.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_a5f9ed84ca9d264b9439.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_a5f9ed84ca9d264b9439.png new file mode 100644 index 0000000000000000000000000000000000000000..1b0cd27f8d8ffebe5ac15973442567c553c5ac21 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_a5f9ed84ca9d264b9439.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_f685f7aab896c2a636a6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_f685f7aab896c2a636a6.png new file mode 100644 index 0000000000000000000000000000000000000000..beba27da42861c8fee97745ecae7be10b98e5df0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2120_f685f7aab896c2a636a6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_1c5e23a118700525a7fd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_1c5e23a118700525a7fd.png new file mode 100644 index 0000000000000000000000000000000000000000..a6a356985f27473f87a604428a59d4a0bf435378 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_1c5e23a118700525a7fd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_40b0d08fdb15bcbe6d76.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_40b0d08fdb15bcbe6d76.png new file mode 100644 index 0000000000000000000000000000000000000000..088076c5fb55552ad25ec8b924dbea1fb5163343 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_40b0d08fdb15bcbe6d76.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_88801f29434a2a3ded1a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_88801f29434a2a3ded1a.png new file mode 100644 index 0000000000000000000000000000000000000000..08ee482a5393cd8f9687ef8f3f431dd2572d475a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_88801f29434a2a3ded1a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_b90bc0dcc518a41e4b7c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_b90bc0dcc518a41e4b7c.png new file mode 100644 index 0000000000000000000000000000000000000000..c91b806c19a0c8eac3e7e7952bcdd69d540b7108 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2128_b90bc0dcc518a41e4b7c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_0358ef62f3e922feee73.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_0358ef62f3e922feee73.png new file mode 100644 index 0000000000000000000000000000000000000000..81785950fe8f7500b255deff846b2e50e8e755c0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_0358ef62f3e922feee73.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_5bc831942544a900b570.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_5bc831942544a900b570.png new file mode 100644 index 0000000000000000000000000000000000000000..1255bc68d9fa66418217b6b9c2ea6cd0c9803a3f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_5bc831942544a900b570.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_961ca855c1854a6642a2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_961ca855c1854a6642a2.png new file mode 100644 index 0000000000000000000000000000000000000000..4af13457cd006f9831d5aa950d328c0bed6c991b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_961ca855c1854a6642a2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_b0e68e79f3ba56f234d1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_b0e68e79f3ba56f234d1.png new file mode 100644 index 0000000000000000000000000000000000000000..5b4e7e53244d27b5edee991b456d23fb73cfd693 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2136_b0e68e79f3ba56f234d1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_1116dc6091dfb2a9a99b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_1116dc6091dfb2a9a99b.png new file mode 100644 index 0000000000000000000000000000000000000000..456f44e7aab1272004e5488919412967b0931311 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_1116dc6091dfb2a9a99b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_33e87beb0aaa1b10bc0b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_33e87beb0aaa1b10bc0b.png new file mode 100644 index 0000000000000000000000000000000000000000..58b93fbe79fec6d1c5b4cd286b3bd17e3e92335c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_33e87beb0aaa1b10bc0b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_3e34196265dc5d0dea47.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_3e34196265dc5d0dea47.png new file mode 100644 index 0000000000000000000000000000000000000000..38c95d8dac682e2e650fb31e1937beb584a56382 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_3e34196265dc5d0dea47.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_dd8f27bdf40cf2d6cea9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_dd8f27bdf40cf2d6cea9.png new file mode 100644 index 0000000000000000000000000000000000000000..33aadc99eab08dd3707954386e6a4652f8612453 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2144_dd8f27bdf40cf2d6cea9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_0772544257fba9b99d85.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_0772544257fba9b99d85.png new file mode 100644 index 0000000000000000000000000000000000000000..74cc47b0053a9415e9934be486f5e7606f5b89a7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_0772544257fba9b99d85.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_7c72c782ab604dee6e7d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_7c72c782ab604dee6e7d.png new file mode 100644 index 0000000000000000000000000000000000000000..f0d14b87444b46dfa4c94bbe05d79f3e73b0b826 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_7c72c782ab604dee6e7d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_9afccb4fc160b21562e7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_9afccb4fc160b21562e7.png new file mode 100644 index 0000000000000000000000000000000000000000..b0e951563ae3f9bcc0bfd914821dd04d83716b31 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_9afccb4fc160b21562e7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_a1a8f442cbc67c6daade.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_a1a8f442cbc67c6daade.png new file mode 100644 index 0000000000000000000000000000000000000000..c76b9a8dcfeb6b1a95a3c18f21cb49c3ce6ec580 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2152_a1a8f442cbc67c6daade.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_42ae5bc916492ef02b4e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_42ae5bc916492ef02b4e.png new file mode 100644 index 0000000000000000000000000000000000000000..bb54ea3d3a105b157b9be351f571ab29ce5101d7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_42ae5bc916492ef02b4e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_802878029984f839da7a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_802878029984f839da7a.png new file mode 100644 index 0000000000000000000000000000000000000000..1a867aa8303084186693f466d3603e1c2c431f3b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_802878029984f839da7a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_d966f9a37b7475161185.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_d966f9a37b7475161185.png new file mode 100644 index 0000000000000000000000000000000000000000..8863493f0d3cc998553efd6fd4dc5fe483f38823 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_d966f9a37b7475161185.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_df30dfecafc8706acc38.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_df30dfecafc8706acc38.png new file mode 100644 index 0000000000000000000000000000000000000000..a58521685f2ea182a882ca96fc668e91a3a03a33 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2160_df30dfecafc8706acc38.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_11433d98fdff95688292.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_11433d98fdff95688292.png new file mode 100644 index 0000000000000000000000000000000000000000..0e65bc4c74d3616966a463c382392217ab9f4ab6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_11433d98fdff95688292.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_7f77a8960cb444700fe1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_7f77a8960cb444700fe1.png new file mode 100644 index 0000000000000000000000000000000000000000..4c4cc57b4912705b1b04eb2ebbcf8c511e6a39a0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_7f77a8960cb444700fe1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_7fa7b07d6e51ffc6ee7c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_7fa7b07d6e51ffc6ee7c.png new file mode 100644 index 0000000000000000000000000000000000000000..a23c241b926da7ffd4f20c10ce793101e988b4e4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_7fa7b07d6e51ffc6ee7c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_ab42f1cb028adcc36624.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_ab42f1cb028adcc36624.png new file mode 100644 index 0000000000000000000000000000000000000000..79ddccef62f21c4874c38e5306bd52451e5b39e6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2168_ab42f1cb028adcc36624.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_4a85055a3c02c45c760a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_4a85055a3c02c45c760a.png new file mode 100644 index 0000000000000000000000000000000000000000..fdc7edf535ecae5e0ec6d084b2f74297e3bec1b6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_4a85055a3c02c45c760a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_4d5eab540def9d6b25f3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_4d5eab540def9d6b25f3.png new file mode 100644 index 0000000000000000000000000000000000000000..38f8f2f7fe034de5f1392f5434ebd95e1cc11b35 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_4d5eab540def9d6b25f3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_7a3a532a14ba9a1b34c2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_7a3a532a14ba9a1b34c2.png new file mode 100644 index 0000000000000000000000000000000000000000..9219ca2daea2ac33ef51b1f6b89830cb57d5ecb9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_7a3a532a14ba9a1b34c2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_cfc28146873310f734c3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_cfc28146873310f734c3.png new file mode 100644 index 0000000000000000000000000000000000000000..2f16cd2cde02efa60bbbd5e2d3b6bfb091583ebb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_216_cfc28146873310f734c3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_0917d80c2d5e3650d3c3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_0917d80c2d5e3650d3c3.png new file mode 100644 index 0000000000000000000000000000000000000000..d66a7b98b789f2a35c88b8181efe7f9e7c40baa4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_0917d80c2d5e3650d3c3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_282f811b6189202a6ebc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_282f811b6189202a6ebc.png new file mode 100644 index 0000000000000000000000000000000000000000..27d87bba490a116b0d356646af4cc01d39013139 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_282f811b6189202a6ebc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_4743122548c8aa7b69c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_4743122548c8aa7b69c0.png new file mode 100644 index 0000000000000000000000000000000000000000..affe8f05a75c954f095a88a21ed5e039b15837c0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_4743122548c8aa7b69c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_91fab3422693e45733b1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_91fab3422693e45733b1.png new file mode 100644 index 0000000000000000000000000000000000000000..99174a29ead11af8d4d075c9feb43687cc0535fb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2176_91fab3422693e45733b1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_1f1d05e973eac5066d87.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_1f1d05e973eac5066d87.png new file mode 100644 index 0000000000000000000000000000000000000000..a3886657f898692c603b432c5d9facdf767772de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_1f1d05e973eac5066d87.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_8bea371a51240befcb0c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_8bea371a51240befcb0c.png new file mode 100644 index 0000000000000000000000000000000000000000..b59ca7d56ad4ac72ffcc4e81aa6c5dd2e804a23a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_8bea371a51240befcb0c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_92af547bd8ce73a91373.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_92af547bd8ce73a91373.png new file mode 100644 index 0000000000000000000000000000000000000000..e5e7385f8511738fc5e6cc885dc09890a6a57082 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_92af547bd8ce73a91373.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_f7d901834bbc876e0496.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_f7d901834bbc876e0496.png new file mode 100644 index 0000000000000000000000000000000000000000..d5dd97569d6b7f983ce0ddfd92322c9f73ca122b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2184_f7d901834bbc876e0496.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_1a9b8a8abe690731327a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_1a9b8a8abe690731327a.png new file mode 100644 index 0000000000000000000000000000000000000000..0b6ce7879cc0428019e31453c8d86408af733519 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_1a9b8a8abe690731327a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_43ded98624cac24bcce2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_43ded98624cac24bcce2.png new file mode 100644 index 0000000000000000000000000000000000000000..ee31dd401ef41840e6f291bf7add4499d5e6915a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_43ded98624cac24bcce2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_ba57f9ac069863aa6a16.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_ba57f9ac069863aa6a16.png new file mode 100644 index 0000000000000000000000000000000000000000..1fae1d26840382212bc431b366057e75f6be68fd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_ba57f9ac069863aa6a16.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_cf71b06b81ea71e6cb38.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_cf71b06b81ea71e6cb38.png new file mode 100644 index 0000000000000000000000000000000000000000..cb1f90662a3a6745a5fb13c9874d8d284b211947 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2192_cf71b06b81ea71e6cb38.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_00596fead5bf8fb0065d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_00596fead5bf8fb0065d.png new file mode 100644 index 0000000000000000000000000000000000000000..840c95464f534fb8471b1d010f6414c3f3bf3a99 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_00596fead5bf8fb0065d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_4dfaff90212e9e27e4d2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_4dfaff90212e9e27e4d2.png new file mode 100644 index 0000000000000000000000000000000000000000..3ab3823117d9af5a99f8f147448722f779cfa984 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_4dfaff90212e9e27e4d2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_c93ac716040789e5a0e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_c93ac716040789e5a0e3.png new file mode 100644 index 0000000000000000000000000000000000000000..7490ae36fd28eb8f81424a5740a840c91c359e52 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_c93ac716040789e5a0e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_ee3e22833ab7e1d894c4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_ee3e22833ab7e1d894c4.png new file mode 100644 index 0000000000000000000000000000000000000000..5c639a2233b1e51a375f775d275bc605b519510d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2200_ee3e22833ab7e1d894c4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_2f73c4c4c50247c1768f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_2f73c4c4c50247c1768f.png new file mode 100644 index 0000000000000000000000000000000000000000..313098bb18039618f8653d2f167667c0738dc376 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_2f73c4c4c50247c1768f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_308d8b2bbe36fb5f30fc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_308d8b2bbe36fb5f30fc.png new file mode 100644 index 0000000000000000000000000000000000000000..a5c2ca7e74a75525e3018a4c0b3253a2f4126119 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_308d8b2bbe36fb5f30fc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_eef023764c90ffd3644e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_eef023764c90ffd3644e.png new file mode 100644 index 0000000000000000000000000000000000000000..53c1e0e72052c468a26e59dc73ed97987113d7f1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_eef023764c90ffd3644e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_ffbfe4f9aca48bda03c2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_ffbfe4f9aca48bda03c2.png new file mode 100644 index 0000000000000000000000000000000000000000..be07f271dbf066f42d92d150f75b5ec5695c6953 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2208_ffbfe4f9aca48bda03c2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_441d862976c9f51b92d6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_441d862976c9f51b92d6.png new file mode 100644 index 0000000000000000000000000000000000000000..dd4421ce55ffd4364faa3a252271d10b5267b10a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_441d862976c9f51b92d6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_5f8a9556edf1586f4491.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_5f8a9556edf1586f4491.png new file mode 100644 index 0000000000000000000000000000000000000000..6b7285cb159cc23ae2d25fe54bfc6ef4e6143411 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_5f8a9556edf1586f4491.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_775468a5cae55b01f9e6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_775468a5cae55b01f9e6.png new file mode 100644 index 0000000000000000000000000000000000000000..9f67166d7ee63acad36dc41d41c2e0b9e055f81d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_775468a5cae55b01f9e6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_8c1ab982496cf82a8f19.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_8c1ab982496cf82a8f19.png new file mode 100644 index 0000000000000000000000000000000000000000..e6b6665a388faa623a194f8bda7182e51dcc3223 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2216_8c1ab982496cf82a8f19.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_13c69897297371dcf8c5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_13c69897297371dcf8c5.png new file mode 100644 index 0000000000000000000000000000000000000000..9f5239da2baf59fb40d85f3b6412fb1fbd1fe3f5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_13c69897297371dcf8c5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_9b460613f0ab2e925269.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_9b460613f0ab2e925269.png new file mode 100644 index 0000000000000000000000000000000000000000..9206878a1a381f24729710fbf174eb806679d3b5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_9b460613f0ab2e925269.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_d3babefe131fdc098ad5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_d3babefe131fdc098ad5.png new file mode 100644 index 0000000000000000000000000000000000000000..139f77f84f68469e40744ac9d7a87abd16c3994b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_d3babefe131fdc098ad5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_ef5660a5a79987882227.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_ef5660a5a79987882227.png new file mode 100644 index 0000000000000000000000000000000000000000..4846bb64dd376420c9ce49dcd3d9e07ca8ce3ed5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2224_ef5660a5a79987882227.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_5428aa3c038948a084d2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_5428aa3c038948a084d2.png new file mode 100644 index 0000000000000000000000000000000000000000..93e5d43be527c20a12105b542db238fca4695061 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_5428aa3c038948a084d2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_810d1b9e7965f4ca6207.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_810d1b9e7965f4ca6207.png new file mode 100644 index 0000000000000000000000000000000000000000..1a929d992256d61937d5070dc219271cc0299565 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_810d1b9e7965f4ca6207.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_d019d6eafe82501f45c5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_d019d6eafe82501f45c5.png new file mode 100644 index 0000000000000000000000000000000000000000..76d3185f345331ed17da816e2fe7a51622233908 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_d019d6eafe82501f45c5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_de35cad4622885a2e7e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_de35cad4622885a2e7e3.png new file mode 100644 index 0000000000000000000000000000000000000000..8a935fdb77964ca029980db831ca09f9099d37c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2232_de35cad4622885a2e7e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_0be5a7d6e1a87c6281ad.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_0be5a7d6e1a87c6281ad.png new file mode 100644 index 0000000000000000000000000000000000000000..d4de2958c00014f988c42bf628fad969aa20b6ac Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_0be5a7d6e1a87c6281ad.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_5e39882221c773c2aaeb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_5e39882221c773c2aaeb.png new file mode 100644 index 0000000000000000000000000000000000000000..84d9e9f8ca3c61e98fcea81082a2173a9324b006 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_5e39882221c773c2aaeb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_e9d3e01b8fb9ddbcc763.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_e9d3e01b8fb9ddbcc763.png new file mode 100644 index 0000000000000000000000000000000000000000..65cf7eca1b2d3887441e5af25792b40a94dd28cb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_e9d3e01b8fb9ddbcc763.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_fc9f1f75957ce5fda00b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_fc9f1f75957ce5fda00b.png new file mode 100644 index 0000000000000000000000000000000000000000..a59597219470d1b8a690ac2b6ecebe8067448aaa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2240_fc9f1f75957ce5fda00b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_20630ae4a09080a1aebc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_20630ae4a09080a1aebc.png new file mode 100644 index 0000000000000000000000000000000000000000..e30ec4dce292cf393744797faefeea879a509e4b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_20630ae4a09080a1aebc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_2662eb4ad8f0124e3af2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_2662eb4ad8f0124e3af2.png new file mode 100644 index 0000000000000000000000000000000000000000..724a54dbb44dcb76c087f5232846f808c10457ea Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_2662eb4ad8f0124e3af2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_4d6ac12b990c4625492f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_4d6ac12b990c4625492f.png new file mode 100644 index 0000000000000000000000000000000000000000..71416e2c8eb1e162b64adfb65ac96ddaa6478a3e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_4d6ac12b990c4625492f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_60ebe036679521533b7c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_60ebe036679521533b7c.png new file mode 100644 index 0000000000000000000000000000000000000000..3e54a9829fce963d2224259f77094b9424ddeaa1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2248_60ebe036679521533b7c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_301d4cf0e8bf493be61e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_301d4cf0e8bf493be61e.png new file mode 100644 index 0000000000000000000000000000000000000000..b25414038ff31a4ffe77a3126e387ff8721f0bfc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_301d4cf0e8bf493be61e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_693eb3a2354f1b571e7c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_693eb3a2354f1b571e7c.png new file mode 100644 index 0000000000000000000000000000000000000000..244b0e08fe59618d1c503a0b53b7045a9a18e238 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_693eb3a2354f1b571e7c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_c41cb95d77436452e8b0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_c41cb95d77436452e8b0.png new file mode 100644 index 0000000000000000000000000000000000000000..cc290c3ae41167fd901e5be72930e282e29e0cca Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_c41cb95d77436452e8b0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_e2e0e2a8e4633ed2e014.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_e2e0e2a8e4633ed2e014.png new file mode 100644 index 0000000000000000000000000000000000000000..5622dea84f2c49a0791cd2b42637b700867225bf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_224_e2e0e2a8e4633ed2e014.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_6030757c8800e8a83428.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_6030757c8800e8a83428.png new file mode 100644 index 0000000000000000000000000000000000000000..72253edbfd4fca9a318d75b6cefdf241847e8e93 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_6030757c8800e8a83428.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_6feffa88023b5b42629e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_6feffa88023b5b42629e.png new file mode 100644 index 0000000000000000000000000000000000000000..de11f93821bb9decf0f22283bd33a12bd9dd4385 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_6feffa88023b5b42629e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_e517c86d4a854fa8f3b6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_e517c86d4a854fa8f3b6.png new file mode 100644 index 0000000000000000000000000000000000000000..f73588fe01c5817edf716ad966f6ddd527bcb422 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_e517c86d4a854fa8f3b6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_f708e90e7fd5277c4fe2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_f708e90e7fd5277c4fe2.png new file mode 100644 index 0000000000000000000000000000000000000000..21496fa11e36794dd3930a7f91c6c01cde25edce Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2256_f708e90e7fd5277c4fe2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_6107b10540fd61346e76.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_6107b10540fd61346e76.png new file mode 100644 index 0000000000000000000000000000000000000000..46771bdb501e0964723eb73edcaebc4d84c46a89 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_6107b10540fd61346e76.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_ab601295b277ed2d8585.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_ab601295b277ed2d8585.png new file mode 100644 index 0000000000000000000000000000000000000000..9f20df8af35a5cceb8b419e9ea52dff6d9ad169d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_ab601295b277ed2d8585.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_d0f444003a54ce1ba603.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_d0f444003a54ce1ba603.png new file mode 100644 index 0000000000000000000000000000000000000000..6fcf957796e514012e0369c2491b6e387856b199 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_d0f444003a54ce1ba603.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_f57b404ae171a2afb73d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_f57b404ae171a2afb73d.png new file mode 100644 index 0000000000000000000000000000000000000000..c00a6cc859a47517e46fd82d903034bed348b2e2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2264_f57b404ae171a2afb73d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_1e1c2cc5fe23888d9a0f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_1e1c2cc5fe23888d9a0f.png new file mode 100644 index 0000000000000000000000000000000000000000..8d86381e29bdc5d4aa0ed15afa21690f6ee29cf6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_1e1c2cc5fe23888d9a0f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_5722a72d5ca3dd7d3435.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_5722a72d5ca3dd7d3435.png new file mode 100644 index 0000000000000000000000000000000000000000..0fd931b21f304446e1e0e3cdde60381b6726a66f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_5722a72d5ca3dd7d3435.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_5b58952985f4c02da378.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_5b58952985f4c02da378.png new file mode 100644 index 0000000000000000000000000000000000000000..58e4f59a85506c6912a4d8028ac32208196dee14 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_5b58952985f4c02da378.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_7bfa00fea9619c31d907.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_7bfa00fea9619c31d907.png new file mode 100644 index 0000000000000000000000000000000000000000..70547560e9cdba184002b354300fca2e93378e0b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2272_7bfa00fea9619c31d907.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_3dc11ed55953ee19931f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_3dc11ed55953ee19931f.png new file mode 100644 index 0000000000000000000000000000000000000000..9abd303f0b84d089cac08ab8a41022ed973a04f1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_3dc11ed55953ee19931f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_6d2411d6fdd40f97df6f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_6d2411d6fdd40f97df6f.png new file mode 100644 index 0000000000000000000000000000000000000000..f9f917568469d6386ecd7517efe6511f75e9cf4d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_6d2411d6fdd40f97df6f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_9613c1b3cfb26096117f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_9613c1b3cfb26096117f.png new file mode 100644 index 0000000000000000000000000000000000000000..8573f63f59be76c304d0911b3c2210742e24a064 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_9613c1b3cfb26096117f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_ea5a63c05b6dc2e76aad.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_ea5a63c05b6dc2e76aad.png new file mode 100644 index 0000000000000000000000000000000000000000..836e76ec9cd77757dd87735c98a731f2c5533985 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2280_ea5a63c05b6dc2e76aad.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_572e255d21901e5a6046.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_572e255d21901e5a6046.png new file mode 100644 index 0000000000000000000000000000000000000000..aa66faf4e34ac29eaef5d6e2d03fd0351afc0dd5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_572e255d21901e5a6046.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_67a013a5a370a35a3059.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_67a013a5a370a35a3059.png new file mode 100644 index 0000000000000000000000000000000000000000..d0d14dcba2c9479d994546cd641aa85efe175c53 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_67a013a5a370a35a3059.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_7286636f68f0a2aa4e30.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_7286636f68f0a2aa4e30.png new file mode 100644 index 0000000000000000000000000000000000000000..a62e9c0082befcf0f01b07c4245f3b4a0340091d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_7286636f68f0a2aa4e30.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_a5cceb7ef5da03ddb014.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_a5cceb7ef5da03ddb014.png new file mode 100644 index 0000000000000000000000000000000000000000..bc8a450fe836b547742716c41c658f94c33ae34e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2288_a5cceb7ef5da03ddb014.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_250d2d9f0b19e4298842.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_250d2d9f0b19e4298842.png new file mode 100644 index 0000000000000000000000000000000000000000..1954e959f4a137a7c655850bcf7d61bc53fec416 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_250d2d9f0b19e4298842.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_47afc8d7e2eb0efcc511.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_47afc8d7e2eb0efcc511.png new file mode 100644 index 0000000000000000000000000000000000000000..bb8683278f60557bdc593b5a73136a7ab373e895 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_47afc8d7e2eb0efcc511.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_cb7397f7fd1ac20d272d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_cb7397f7fd1ac20d272d.png new file mode 100644 index 0000000000000000000000000000000000000000..bf433e2b25b71c0eca06f1e0ea7364d57205a1ea Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_cb7397f7fd1ac20d272d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_e87577dcb5ac8c5c2a2c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_e87577dcb5ac8c5c2a2c.png new file mode 100644 index 0000000000000000000000000000000000000000..496f09758d4f499fa65b4ee7f78e0e398b6bfc69 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2296_e87577dcb5ac8c5c2a2c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_2e6facfca2619f0d7cb5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_2e6facfca2619f0d7cb5.png new file mode 100644 index 0000000000000000000000000000000000000000..47c67ea24b672e1dbb7132f7ff671725c8d0f0da Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_2e6facfca2619f0d7cb5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_4cd5bd883752c4c9c8c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_4cd5bd883752c4c9c8c0.png new file mode 100644 index 0000000000000000000000000000000000000000..53e0dd510bdbcc7b8cbf0d6693aed64da4cf5383 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_4cd5bd883752c4c9c8c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_5c183443d040ea97fa1b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_5c183443d040ea97fa1b.png new file mode 100644 index 0000000000000000000000000000000000000000..79a4ed65772a3342e2076fb498c162835b8f3ed2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_5c183443d040ea97fa1b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_e90e75900878a3127495.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_e90e75900878a3127495.png new file mode 100644 index 0000000000000000000000000000000000000000..c60fa97c769c63e95069babd9be700f573066df0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2304_e90e75900878a3127495.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_21c7c0440b0b11b2d491.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_21c7c0440b0b11b2d491.png new file mode 100644 index 0000000000000000000000000000000000000000..9b048f58a28a9966455f34cfd5218a2cc7aa415d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_21c7c0440b0b11b2d491.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_52029317168919eb0da8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_52029317168919eb0da8.png new file mode 100644 index 0000000000000000000000000000000000000000..95a522efadb54abd3ee3f08d058b8a76c7dae07b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_52029317168919eb0da8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_9c89f883bf5fcc238280.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_9c89f883bf5fcc238280.png new file mode 100644 index 0000000000000000000000000000000000000000..b23e474ba16ef4a5efe4db7b2700b3e8752c98fc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_9c89f883bf5fcc238280.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_e9126f14486264f97c5b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_e9126f14486264f97c5b.png new file mode 100644 index 0000000000000000000000000000000000000000..fa7a1926f787c994d78ca9900dbded5f101f516a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2312_e9126f14486264f97c5b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_39c700c0233bf9ec1887.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_39c700c0233bf9ec1887.png new file mode 100644 index 0000000000000000000000000000000000000000..b8ed4243236e7b8cfb533ac8401dd7c426c05028 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_39c700c0233bf9ec1887.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_3e0ad5d834a85111111c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_3e0ad5d834a85111111c.png new file mode 100644 index 0000000000000000000000000000000000000000..ca46250739e690f2d9f75dbc7f7b66f0d9c889a5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_3e0ad5d834a85111111c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_68a147e55429db2d8cd5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_68a147e55429db2d8cd5.png new file mode 100644 index 0000000000000000000000000000000000000000..c6a4728d5941fe43cd2e259ab76e675a7bee2450 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_68a147e55429db2d8cd5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_84ca29059a3b9b5905fc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_84ca29059a3b9b5905fc.png new file mode 100644 index 0000000000000000000000000000000000000000..8114419082ee721662a865c1acb967e9b1cece2f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2320_84ca29059a3b9b5905fc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_0dafda20d7536a74a759.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_0dafda20d7536a74a759.png new file mode 100644 index 0000000000000000000000000000000000000000..855d80bbf08b3b309518c1e3a0db5cbeb5b3b3d2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_0dafda20d7536a74a759.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_26e2b28b1b2c040caf0a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_26e2b28b1b2c040caf0a.png new file mode 100644 index 0000000000000000000000000000000000000000..d8c5e230c55af8b4eac2e617745264d13004c65b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_26e2b28b1b2c040caf0a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_62f7b60d0949855bbdda.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_62f7b60d0949855bbdda.png new file mode 100644 index 0000000000000000000000000000000000000000..9b6f9d607f67ce3939caa9f43cf0811c9da81126 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_62f7b60d0949855bbdda.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_80c49839f74d15e546e2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_80c49839f74d15e546e2.png new file mode 100644 index 0000000000000000000000000000000000000000..90feb10e23de5123936b469504b9dc5fc046c307 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2328_80c49839f74d15e546e2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_713378cc3efc25bf233a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_713378cc3efc25bf233a.png new file mode 100644 index 0000000000000000000000000000000000000000..c42d96f7225efa424d27131a5f6d23dbca8114e6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_713378cc3efc25bf233a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_a060116adb492f4b9a59.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_a060116adb492f4b9a59.png new file mode 100644 index 0000000000000000000000000000000000000000..452b66b53b881108758e15999f656d525e1b53ea Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_a060116adb492f4b9a59.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_c57b0c82f494de77f729.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_c57b0c82f494de77f729.png new file mode 100644 index 0000000000000000000000000000000000000000..5884ec045947b3435081796c91009ff79cba7329 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_c57b0c82f494de77f729.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_c7a422f0918a7e63fef3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_c7a422f0918a7e63fef3.png new file mode 100644 index 0000000000000000000000000000000000000000..0492046ebcc0e97621638876b3a28ce05f77f1cd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_232_c7a422f0918a7e63fef3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_301a887d407c4ac4a9e8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_301a887d407c4ac4a9e8.png new file mode 100644 index 0000000000000000000000000000000000000000..613befc900c7c625b574b2467885ee2a82ad46f5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_301a887d407c4ac4a9e8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_354d7f9111304540102a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_354d7f9111304540102a.png new file mode 100644 index 0000000000000000000000000000000000000000..1d6328cc87862c87b34de1a7a9e48417afe692aa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_354d7f9111304540102a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_a5b295c1cf8890d267c2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_a5b295c1cf8890d267c2.png new file mode 100644 index 0000000000000000000000000000000000000000..21270db745acfffdf055867999058e829d6a747b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_a5b295c1cf8890d267c2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_ae700d32677535960099.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_ae700d32677535960099.png new file mode 100644 index 0000000000000000000000000000000000000000..6b0663d3a6fa508af244413f398f766dcebf4876 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2336_ae700d32677535960099.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_b0129eec4a00fb5d2469.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_b0129eec4a00fb5d2469.png new file mode 100644 index 0000000000000000000000000000000000000000..b3f7981b954cfc2d244e92d352777614f6ac15b8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_b0129eec4a00fb5d2469.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_d627b8b33c228c221574.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_d627b8b33c228c221574.png new file mode 100644 index 0000000000000000000000000000000000000000..1592504ad39621bf92a89cab70ece9b7e0ccec1a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_d627b8b33c228c221574.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_e060b93c7cc983117765.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_e060b93c7cc983117765.png new file mode 100644 index 0000000000000000000000000000000000000000..629f6c9c6ebc3ae719495097eb85d1ab042bfa5f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_e060b93c7cc983117765.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_f26688d7e9d47ebeba3b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_f26688d7e9d47ebeba3b.png new file mode 100644 index 0000000000000000000000000000000000000000..7215ff31e72fd79cd963e8844984a2facaffb7be Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2344_f26688d7e9d47ebeba3b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_c61b0d4f5e5ee8adb21f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_c61b0d4f5e5ee8adb21f.png new file mode 100644 index 0000000000000000000000000000000000000000..919796b767dbf003016218c56c0767f0c9bc4a33 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_c61b0d4f5e5ee8adb21f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_c77b9bc3c6982cc22106.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_c77b9bc3c6982cc22106.png new file mode 100644 index 0000000000000000000000000000000000000000..a09e4d9f22119367688c09698b6b59bf5f1e856e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_c77b9bc3c6982cc22106.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_ce593cd883ca40dee3d7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_ce593cd883ca40dee3d7.png new file mode 100644 index 0000000000000000000000000000000000000000..b96b781f6a0a18710b782391dec15c1b575676de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_ce593cd883ca40dee3d7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_e9cbdb785c02cd01bf26.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_e9cbdb785c02cd01bf26.png new file mode 100644 index 0000000000000000000000000000000000000000..232fe69b2a57effd4b2cd7d92f1c773807059952 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2352_e9cbdb785c02cd01bf26.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_0a11cdad43876d67f6b1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_0a11cdad43876d67f6b1.png new file mode 100644 index 0000000000000000000000000000000000000000..cd11f81cde872fbb31f2748f05567024e88dd66b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_0a11cdad43876d67f6b1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_4d37f738f3ac19d459df.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_4d37f738f3ac19d459df.png new file mode 100644 index 0000000000000000000000000000000000000000..aa6d2681fff9168a4f809510a9d09720cf0f2dc5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_4d37f738f3ac19d459df.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_945a8790f028d3a3a4ac.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_945a8790f028d3a3a4ac.png new file mode 100644 index 0000000000000000000000000000000000000000..e6f460c53f73c81d1633e219bfcb1d6d833c580b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_945a8790f028d3a3a4ac.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_9df37ef1c70edfc470f2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_9df37ef1c70edfc470f2.png new file mode 100644 index 0000000000000000000000000000000000000000..d64dbdca3a11e25ba2192c6b26e98b00bd672a93 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2360_9df37ef1c70edfc470f2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_0b60b02ca6831735feee.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_0b60b02ca6831735feee.png new file mode 100644 index 0000000000000000000000000000000000000000..a0890a0a54efcb837c97b159d50b8ec5abee0744 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_0b60b02ca6831735feee.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_342e28a8b292c6288351.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_342e28a8b292c6288351.png new file mode 100644 index 0000000000000000000000000000000000000000..1dec7119a0babde3d42223aa3d10cdf033857d7b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_342e28a8b292c6288351.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_d3688889a7a8ef78609d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_d3688889a7a8ef78609d.png new file mode 100644 index 0000000000000000000000000000000000000000..758aa290432772c643c9561a102c86d36ce18ded Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_d3688889a7a8ef78609d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_e455e3cf2c345fdc0326.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_e455e3cf2c345fdc0326.png new file mode 100644 index 0000000000000000000000000000000000000000..1a9c4285e6960b97deccf0a48575ed1319e66948 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2368_e455e3cf2c345fdc0326.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_11c406e93bca8b9b3bca.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_11c406e93bca8b9b3bca.png new file mode 100644 index 0000000000000000000000000000000000000000..c382d0d0a2f8f47534d1f5233c8ea145849be3e0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_11c406e93bca8b9b3bca.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_bc22907973133c98cdde.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_bc22907973133c98cdde.png new file mode 100644 index 0000000000000000000000000000000000000000..1c023fe666f3663fa7126118e1a5de39c62630b1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_bc22907973133c98cdde.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_cc2cb6db5aeb4f4ce3e7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_cc2cb6db5aeb4f4ce3e7.png new file mode 100644 index 0000000000000000000000000000000000000000..6279be67a7323ae9388b51d0f0380d4cf25e0371 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_cc2cb6db5aeb4f4ce3e7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_e2203a4cdfa94cd450dc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_e2203a4cdfa94cd450dc.png new file mode 100644 index 0000000000000000000000000000000000000000..e20106761524826efede644f39d13145fcf7e7eb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2376_e2203a4cdfa94cd450dc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_0b1a1c25ca7c935944d8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_0b1a1c25ca7c935944d8.png new file mode 100644 index 0000000000000000000000000000000000000000..0943941a91fd39db5a9eed5898a1439ce2fd3905 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_0b1a1c25ca7c935944d8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_34e253f269b378eb15d6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_34e253f269b378eb15d6.png new file mode 100644 index 0000000000000000000000000000000000000000..bacb3fd264889c8257449d617954d9bed85da6fa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_34e253f269b378eb15d6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_81c5a876097ae538156d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_81c5a876097ae538156d.png new file mode 100644 index 0000000000000000000000000000000000000000..85cb61741ae111fdacf01c42cf49bea188366218 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_81c5a876097ae538156d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_d4b3165970b466047f71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_d4b3165970b466047f71.png new file mode 100644 index 0000000000000000000000000000000000000000..271f811625716475714ce1fa9e9f806453adfe96 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2384_d4b3165970b466047f71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_15295da0dc5bd19625bd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_15295da0dc5bd19625bd.png new file mode 100644 index 0000000000000000000000000000000000000000..0915fcc982691df73ce471edc881c3b5c0c69e26 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_15295da0dc5bd19625bd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_1c61e21e1cd4c686eaaf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_1c61e21e1cd4c686eaaf.png new file mode 100644 index 0000000000000000000000000000000000000000..01f51a2e94372b423ed74ca92e75c9f30ab8a7bc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_1c61e21e1cd4c686eaaf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_884c2d28486ad0ad23ae.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_884c2d28486ad0ad23ae.png new file mode 100644 index 0000000000000000000000000000000000000000..a9e053a445765d21eecf83af19dcb4b6a6116c29 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_884c2d28486ad0ad23ae.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_ab6620fbc899727dc772.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_ab6620fbc899727dc772.png new file mode 100644 index 0000000000000000000000000000000000000000..c9bbc6b66ad4200cb9722783e8a7ee93be721387 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2392_ab6620fbc899727dc772.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_550889b0c84a4a159425.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_550889b0c84a4a159425.png new file mode 100644 index 0000000000000000000000000000000000000000..0ecdf3096b61d2873b9024c5f3d6a34ea35620dc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_550889b0c84a4a159425.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_689387d208c87e82a9d4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_689387d208c87e82a9d4.png new file mode 100644 index 0000000000000000000000000000000000000000..62f1f1c9205a3bb70568ef333b4d403cee3c01f3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_689387d208c87e82a9d4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_8744e2e9594e95a56a40.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_8744e2e9594e95a56a40.png new file mode 100644 index 0000000000000000000000000000000000000000..caef780b6fc10558bee2daedab9ba319dc4c93d8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_8744e2e9594e95a56a40.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_edf7e219837bb0398a66.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_edf7e219837bb0398a66.png new file mode 100644 index 0000000000000000000000000000000000000000..a15057e141a893245325a4f72bc71b9eba4b9b2f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_2400_edf7e219837bb0398a66.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_131f48b2cfb4ebcfb628.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_131f48b2cfb4ebcfb628.png new file mode 100644 index 0000000000000000000000000000000000000000..050352f0019d6af1ea47bd0cd1113b2cba72152b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_131f48b2cfb4ebcfb628.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_4363a80dfcc1b540ae2c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_4363a80dfcc1b540ae2c.png new file mode 100644 index 0000000000000000000000000000000000000000..35e4e38ff9a977d2b5278eb94874d24015e295a2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_4363a80dfcc1b540ae2c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_b17212d3811e99b0c92e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_b17212d3811e99b0c92e.png new file mode 100644 index 0000000000000000000000000000000000000000..b9a575efaa65c5accbff0b14ba1fdce8ce880277 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_b17212d3811e99b0c92e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_cf699d9c0706f5fbf1d1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_cf699d9c0706f5fbf1d1.png new file mode 100644 index 0000000000000000000000000000000000000000..2449a51bd809826d03182b795a419c3e74a7431e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_240_cf699d9c0706f5fbf1d1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_57792db981d24437505d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_57792db981d24437505d.png new file mode 100644 index 0000000000000000000000000000000000000000..4dcc813cd965c60e7bdb0be2f152132254b687fb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_57792db981d24437505d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_84e81851db303d973735.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_84e81851db303d973735.png new file mode 100644 index 0000000000000000000000000000000000000000..c7e12f1a76acae06954b33dad67e392f9cbfdc99 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_84e81851db303d973735.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_981e491cb9b69cf79bc6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_981e491cb9b69cf79bc6.png new file mode 100644 index 0000000000000000000000000000000000000000..4dc0cb8b49b453a222d2eceb87881c4efe3e44b9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_981e491cb9b69cf79bc6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_9e9ae0e3b6539cacbcfd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_9e9ae0e3b6539cacbcfd.png new file mode 100644 index 0000000000000000000000000000000000000000..91b6b24678e4038cd9f511a8b3797c399aabbe95 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_248_9e9ae0e3b6539cacbcfd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_0684dbe8f6eb82a2da7f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_0684dbe8f6eb82a2da7f.png new file mode 100644 index 0000000000000000000000000000000000000000..1fb02a33b8c7a518f61b2980018787261171dbcd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_0684dbe8f6eb82a2da7f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_5a5b54a523e7e8a202d4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_5a5b54a523e7e8a202d4.png new file mode 100644 index 0000000000000000000000000000000000000000..c67413a241e5e1874f14037d6a9300433c2cdecc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_5a5b54a523e7e8a202d4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_68fac8d4cdcb87c0351c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_68fac8d4cdcb87c0351c.png new file mode 100644 index 0000000000000000000000000000000000000000..259042823a8e9d14a3951900db2974b0b76d00c3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_68fac8d4cdcb87c0351c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_e0d04b7701f5a5d35a98.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_e0d04b7701f5a5d35a98.png new file mode 100644 index 0000000000000000000000000000000000000000..05e3e54b273bbead8fdf29e47637d9f024e776bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_24_e0d04b7701f5a5d35a98.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_003e58e75a4b35087fc3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_003e58e75a4b35087fc3.png new file mode 100644 index 0000000000000000000000000000000000000000..051eae77abcd48c98d162b389ccaa3b237a28d58 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_003e58e75a4b35087fc3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_398994785799033e6506.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_398994785799033e6506.png new file mode 100644 index 0000000000000000000000000000000000000000..2245c1e6bf18b9ef5e995c155d54f9860d9320eb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_398994785799033e6506.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_6e977e09b8e28e89d88b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_6e977e09b8e28e89d88b.png new file mode 100644 index 0000000000000000000000000000000000000000..20140053a479debb45d3be02802ee94da4540cc0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_6e977e09b8e28e89d88b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_e0fd66e23b03e8fb0137.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_e0fd66e23b03e8fb0137.png new file mode 100644 index 0000000000000000000000000000000000000000..4129792c45daa4b6b358f29f16af1d59bc6718d2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_256_e0fd66e23b03e8fb0137.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_0ba4a6c56127f845b572.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_0ba4a6c56127f845b572.png new file mode 100644 index 0000000000000000000000000000000000000000..e907ce8da50b707a6258a646984c625ab8bf9f01 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_0ba4a6c56127f845b572.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_50b6b68312c4f0c0f919.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_50b6b68312c4f0c0f919.png new file mode 100644 index 0000000000000000000000000000000000000000..6434187a8d6be5f3aa0a2b938603237324013c7b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_50b6b68312c4f0c0f919.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_be323385875893b62402.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_be323385875893b62402.png new file mode 100644 index 0000000000000000000000000000000000000000..a1988e572ab47c1f8b39a143c803d3204fcc06c1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_be323385875893b62402.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_df9f8e90426ed7bffb77.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_df9f8e90426ed7bffb77.png new file mode 100644 index 0000000000000000000000000000000000000000..95f53cd080eb2930423d8a0f2dfe415e35a06571 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_264_df9f8e90426ed7bffb77.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_2787282524c47ad5cb18.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_2787282524c47ad5cb18.png new file mode 100644 index 0000000000000000000000000000000000000000..a216c7eb0e183d5891a4327afef8226566c5dd23 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_2787282524c47ad5cb18.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_5f6299d67a2c504b2dcc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_5f6299d67a2c504b2dcc.png new file mode 100644 index 0000000000000000000000000000000000000000..a24ca59a346574f236592a53ff0923cd41de3ca5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_5f6299d67a2c504b2dcc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_c2159fc2ce0f2913430b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_c2159fc2ce0f2913430b.png new file mode 100644 index 0000000000000000000000000000000000000000..ccbde511157f61321108ecf2eb72b8eb798ac19a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_c2159fc2ce0f2913430b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_cd804946c37522db1906.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_cd804946c37522db1906.png new file mode 100644 index 0000000000000000000000000000000000000000..93c8d062990072040f823dd50e7cf854bf63106d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_272_cd804946c37522db1906.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_4af7c3c5f9585a30429c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_4af7c3c5f9585a30429c.png new file mode 100644 index 0000000000000000000000000000000000000000..9623dc42ef8e16d77b7dfcb2f616665172d97ae1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_4af7c3c5f9585a30429c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_558ec026a5f3b0c54aab.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_558ec026a5f3b0c54aab.png new file mode 100644 index 0000000000000000000000000000000000000000..586da9ae496e756789f7031951da61287644f9d7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_558ec026a5f3b0c54aab.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_a60cb3ddac08d073d30c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_a60cb3ddac08d073d30c.png new file mode 100644 index 0000000000000000000000000000000000000000..a73e1e616fe9e5b4888171aabf85b483fadf3772 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_a60cb3ddac08d073d30c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_c9a2c308a53c060e2f9e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_c9a2c308a53c060e2f9e.png new file mode 100644 index 0000000000000000000000000000000000000000..5d9e086f4a3697d37efd107e93b4c2cd76a55d49 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_280_c9a2c308a53c060e2f9e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_2a46f6447f317761e631.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_2a46f6447f317761e631.png new file mode 100644 index 0000000000000000000000000000000000000000..1a2b1343b832155588449c69995e34c8f7a10310 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_2a46f6447f317761e631.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_7092f5d1e02e7ebba8fb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_7092f5d1e02e7ebba8fb.png new file mode 100644 index 0000000000000000000000000000000000000000..ecbf7aa5de7d628a94f090c83ddda2f9cb32d52a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_7092f5d1e02e7ebba8fb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_d322f4dde2e9d2d4c2d0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_d322f4dde2e9d2d4c2d0.png new file mode 100644 index 0000000000000000000000000000000000000000..0ff23db007f5422392c80dde5538bd8d6286c869 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_d322f4dde2e9d2d4c2d0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_e8cc62c8cd0593a12258.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_e8cc62c8cd0593a12258.png new file mode 100644 index 0000000000000000000000000000000000000000..f0433fe64fab17a8f18edfbb237b6e1dc30a9bd9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_288_e8cc62c8cd0593a12258.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_1ed73a63f24513e30257.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_1ed73a63f24513e30257.png new file mode 100644 index 0000000000000000000000000000000000000000..78da3e27e759762070e1a6a6d5615e834d5633b3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_1ed73a63f24513e30257.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_88dc791f343412ad2af1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_88dc791f343412ad2af1.png new file mode 100644 index 0000000000000000000000000000000000000000..c2d618e16d0572664ccf68d94be427755faa11bf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_88dc791f343412ad2af1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_9e30465455fb3e14420c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_9e30465455fb3e14420c.png new file mode 100644 index 0000000000000000000000000000000000000000..718ad3a07ab0eec14efa8910839febb317e4d771 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_9e30465455fb3e14420c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_ce669110321002a765f4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_ce669110321002a765f4.png new file mode 100644 index 0000000000000000000000000000000000000000..9d4503aa9c047ef5604627532b2c0442a962ef15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_296_ce669110321002a765f4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_408941d3a9aad78e3825.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_408941d3a9aad78e3825.png new file mode 100644 index 0000000000000000000000000000000000000000..76cf381598abc4313df03a8f6ab53aaf20fa18af Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_408941d3a9aad78e3825.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_8cb43723394c9698f98b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_8cb43723394c9698f98b.png new file mode 100644 index 0000000000000000000000000000000000000000..fbfb260ffdc09892511c91cb903689a403f47779 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_8cb43723394c9698f98b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_acbea64c9e5d9aa9013b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_acbea64c9e5d9aa9013b.png new file mode 100644 index 0000000000000000000000000000000000000000..108f3d240d0ce222119c8466c29dfa16de3a4563 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_acbea64c9e5d9aa9013b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_ce2d8ab42d2884a6a507.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_ce2d8ab42d2884a6a507.png new file mode 100644 index 0000000000000000000000000000000000000000..7670e0077a932a3189c2a0699ebb438fe8a6824b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_304_ce2d8ab42d2884a6a507.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_3f17f5fe28a713544173.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_3f17f5fe28a713544173.png new file mode 100644 index 0000000000000000000000000000000000000000..53a1660fd8acc4fa79a089f0986095b2efb9e49e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_3f17f5fe28a713544173.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_8d287303a5c0f6c9b182.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_8d287303a5c0f6c9b182.png new file mode 100644 index 0000000000000000000000000000000000000000..586abea12c468c647e2fbfe4485b1ac1f208e12c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_8d287303a5c0f6c9b182.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_ae0fbaa0b9e169ccfaf7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_ae0fbaa0b9e169ccfaf7.png new file mode 100644 index 0000000000000000000000000000000000000000..d8bcb13f23a5223647b354c1e38372656c93bc87 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_ae0fbaa0b9e169ccfaf7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_d97fb722dbbfe3980d7a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_d97fb722dbbfe3980d7a.png new file mode 100644 index 0000000000000000000000000000000000000000..8287a7f4beb0ef6e80a6b451eb16678fb800aec1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_312_d97fb722dbbfe3980d7a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_59eb061cba639d911944.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_59eb061cba639d911944.png new file mode 100644 index 0000000000000000000000000000000000000000..be2a80d2368889affbfabefc05e69ea732c9f5b1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_59eb061cba639d911944.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_6fac76e86a9c94c7429f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_6fac76e86a9c94c7429f.png new file mode 100644 index 0000000000000000000000000000000000000000..6d410ae2c9054a57b69763432f42217165bc2bc8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_6fac76e86a9c94c7429f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_787669ccfd6f2189c2e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_787669ccfd6f2189c2e9.png new file mode 100644 index 0000000000000000000000000000000000000000..8b991deb1509b9f3d5e5ef0a865fbe73ff3531f6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_787669ccfd6f2189c2e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_ca6b8df2ff2f328c13ab.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_ca6b8df2ff2f328c13ab.png new file mode 100644 index 0000000000000000000000000000000000000000..e74f8d77727ac450f4991fdc4cc7e3919eab91f5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_320_ca6b8df2ff2f328c13ab.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_5ee531180049cce5568d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_5ee531180049cce5568d.png new file mode 100644 index 0000000000000000000000000000000000000000..5a83c9de3f515af3d4c38eb7b727e8285eac6a17 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_5ee531180049cce5568d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_9a95e0071e58dbf2382e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_9a95e0071e58dbf2382e.png new file mode 100644 index 0000000000000000000000000000000000000000..f698208a814389a21ca6ee5abf7897804b9d3364 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_9a95e0071e58dbf2382e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_a295737d90d34f9422e2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_a295737d90d34f9422e2.png new file mode 100644 index 0000000000000000000000000000000000000000..c18145f81a7b3259731a1d9d949ea7deeadc126f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_a295737d90d34f9422e2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_d261d0f406cd6d609cb4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_d261d0f406cd6d609cb4.png new file mode 100644 index 0000000000000000000000000000000000000000..71b65b35a8dab49ae2e6386f868686557ad462a4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_328_d261d0f406cd6d609cb4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_603e1acd8de551001b1b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_603e1acd8de551001b1b.png new file mode 100644 index 0000000000000000000000000000000000000000..e57ee0e645b7098dd234a76c9ec4f5a858216c58 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_603e1acd8de551001b1b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_7ece7c36041231bd9830.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_7ece7c36041231bd9830.png new file mode 100644 index 0000000000000000000000000000000000000000..01297da511cf7bc42e5bd7bcd78701fd2e022c35 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_7ece7c36041231bd9830.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_caeb7e11117ca44da4c8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_caeb7e11117ca44da4c8.png new file mode 100644 index 0000000000000000000000000000000000000000..a7afa850524eebba5f55379f360787beb6db0865 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_caeb7e11117ca44da4c8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_d076df0f0d397a793093.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_d076df0f0d397a793093.png new file mode 100644 index 0000000000000000000000000000000000000000..2ca10ad560e609e2eaf2984f9603b480b1db2eb3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_32_d076df0f0d397a793093.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_0ffc6176fb6d7e1c0839.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_0ffc6176fb6d7e1c0839.png new file mode 100644 index 0000000000000000000000000000000000000000..db2c5fb2785027f966eaf6b168edb1d245849472 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_0ffc6176fb6d7e1c0839.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_a39fd3fa08d829b242c5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_a39fd3fa08d829b242c5.png new file mode 100644 index 0000000000000000000000000000000000000000..602ba99ebc31770ab35cfbef50905c0a8796adb0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_a39fd3fa08d829b242c5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_d3ae5109418d504e15a5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_d3ae5109418d504e15a5.png new file mode 100644 index 0000000000000000000000000000000000000000..f08c1361c25efe3365408eb06c3654a238d1217f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_d3ae5109418d504e15a5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_db973e73f7779e6592b8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_db973e73f7779e6592b8.png new file mode 100644 index 0000000000000000000000000000000000000000..5a75cdffff8306918610d6e04e241eb79d816187 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_336_db973e73f7779e6592b8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_6aa625d41ec15aca550a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_6aa625d41ec15aca550a.png new file mode 100644 index 0000000000000000000000000000000000000000..7c3fe391d211426ad3ce632173eff387085c34a5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_6aa625d41ec15aca550a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_cea16b4401e675df5e86.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_cea16b4401e675df5e86.png new file mode 100644 index 0000000000000000000000000000000000000000..02575bad175ac14f8a4d93b6f2c9161515d8c2cb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_cea16b4401e675df5e86.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_f5043cd63cbfcc5b803e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_f5043cd63cbfcc5b803e.png new file mode 100644 index 0000000000000000000000000000000000000000..d05ff097eb76e49058da0c0c3d88c68039ca2f26 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_f5043cd63cbfcc5b803e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_f73ffb4d2e046ab554c4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_f73ffb4d2e046ab554c4.png new file mode 100644 index 0000000000000000000000000000000000000000..0d763849def05b0c561bcb17b62cf3fb271f1187 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_344_f73ffb4d2e046ab554c4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_3a044ff8368fa9c583c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_3a044ff8368fa9c583c0.png new file mode 100644 index 0000000000000000000000000000000000000000..26d06f78a19cce6485570e4419f5e78d0dcd9609 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_3a044ff8368fa9c583c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_6788b6099bc68d956e71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_6788b6099bc68d956e71.png new file mode 100644 index 0000000000000000000000000000000000000000..6140f61b8613be4837960d38cdfb3339857e667f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_6788b6099bc68d956e71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_816057c5787d207de5ff.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_816057c5787d207de5ff.png new file mode 100644 index 0000000000000000000000000000000000000000..1fa9142f5f0ef0f9fc2c2a9b32745c406ebe2c69 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_816057c5787d207de5ff.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_8c3b42ad97f04b9342e2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_8c3b42ad97f04b9342e2.png new file mode 100644 index 0000000000000000000000000000000000000000..708c4a01a4778e229e6523dafb382f6714a36523 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_352_8c3b42ad97f04b9342e2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_9a31898c9077cdd9a41d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_9a31898c9077cdd9a41d.png new file mode 100644 index 0000000000000000000000000000000000000000..0bb2a4d8c8b782d504004a2c0109015e8c388c27 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_9a31898c9077cdd9a41d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_c20dcd4a948eaf4fa197.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_c20dcd4a948eaf4fa197.png new file mode 100644 index 0000000000000000000000000000000000000000..fe137627034212092b4cb3d9369d6af8d307977a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_c20dcd4a948eaf4fa197.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_c8519aaa69a9bd44cd79.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_c8519aaa69a9bd44cd79.png new file mode 100644 index 0000000000000000000000000000000000000000..cac8116545709bbc482f734b6c86a78057e74386 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_c8519aaa69a9bd44cd79.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_e2e4e89dbfed8848cd88.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_e2e4e89dbfed8848cd88.png new file mode 100644 index 0000000000000000000000000000000000000000..4ebbb0c44168fec6324831d7fce1e06471cca3bd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_360_e2e4e89dbfed8848cd88.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_718865898338ad2310bd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_718865898338ad2310bd.png new file mode 100644 index 0000000000000000000000000000000000000000..b9fc15c62e7fda84902025e7bd84659291e4e95e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_718865898338ad2310bd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_72cbe5ebdd0210e3d6fe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_72cbe5ebdd0210e3d6fe.png new file mode 100644 index 0000000000000000000000000000000000000000..c0891f2507289b884237209485f43e413530d4a1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_72cbe5ebdd0210e3d6fe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_e3360ff6b5d76e8dac77.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_e3360ff6b5d76e8dac77.png new file mode 100644 index 0000000000000000000000000000000000000000..2ca64584c79fcc5dd08127768c4605d48396d241 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_e3360ff6b5d76e8dac77.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_f4da615e9afd2c2cd8b4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_f4da615e9afd2c2cd8b4.png new file mode 100644 index 0000000000000000000000000000000000000000..16e597c4dac1af112a560874db89ae5453f6bf7b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_368_f4da615e9afd2c2cd8b4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_05d73dd8573520eeccd6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_05d73dd8573520eeccd6.png new file mode 100644 index 0000000000000000000000000000000000000000..525a0e52197b0fc6c377b581281e68614ec4f502 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_05d73dd8573520eeccd6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_1c6e0b1c9c4a02e2487e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_1c6e0b1c9c4a02e2487e.png new file mode 100644 index 0000000000000000000000000000000000000000..527a9c4767cfb15c2b3a11244c24f28c50006c4a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_1c6e0b1c9c4a02e2487e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_c945a66b43ba7bfe4fc4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_c945a66b43ba7bfe4fc4.png new file mode 100644 index 0000000000000000000000000000000000000000..a37aea13c88a6ab0969b966a739543bf9f702a49 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_c945a66b43ba7bfe4fc4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_e09462e3bcde1058d20b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_e09462e3bcde1058d20b.png new file mode 100644 index 0000000000000000000000000000000000000000..2748714c9aa6f4219c5ca1a0b25a46bb0ce3a845 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_376_e09462e3bcde1058d20b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_11cb9d1bc6f6a97d0955.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_11cb9d1bc6f6a97d0955.png new file mode 100644 index 0000000000000000000000000000000000000000..ee7ba05d9aab0a96cb890e13c0ef7c580a1f4cc8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_11cb9d1bc6f6a97d0955.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_155d4ae5c7fbde37d2d4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_155d4ae5c7fbde37d2d4.png new file mode 100644 index 0000000000000000000000000000000000000000..9cb4f17b87f4bc1801ca88cf171026ed808d5849 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_155d4ae5c7fbde37d2d4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_7a702633095b41268383.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_7a702633095b41268383.png new file mode 100644 index 0000000000000000000000000000000000000000..234caf5cd4a2099614da4e387b9410c086254412 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_7a702633095b41268383.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_905222ac5248c3db9c7f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_905222ac5248c3db9c7f.png new file mode 100644 index 0000000000000000000000000000000000000000..4ec4dfb277601deba821f209ff7ce402bd914214 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_384_905222ac5248c3db9c7f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_6c8f25f05cf96b671ec4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_6c8f25f05cf96b671ec4.png new file mode 100644 index 0000000000000000000000000000000000000000..4f4d25572570c32be269001e9375367ef1754fe3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_6c8f25f05cf96b671ec4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_7d995d67c424df992085.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_7d995d67c424df992085.png new file mode 100644 index 0000000000000000000000000000000000000000..5aa8457e043a87def945907ea2ff09e51b3852d6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_7d995d67c424df992085.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_86c67c37eda7ead98c90.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_86c67c37eda7ead98c90.png new file mode 100644 index 0000000000000000000000000000000000000000..4653ac27009ac4b72ed5e399bad4d34481504398 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_86c67c37eda7ead98c90.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_cba110247f071a63aa27.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_cba110247f071a63aa27.png new file mode 100644 index 0000000000000000000000000000000000000000..03829cb90150e48b132c3f65808fb5d583601b00 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_392_cba110247f071a63aa27.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_1a34069e82abe9147618.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_1a34069e82abe9147618.png new file mode 100644 index 0000000000000000000000000000000000000000..6353195c8f45dc5afea78fc5f535b91dbbb24a1d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_1a34069e82abe9147618.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_b541dd5a4efd2e7f476e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_b541dd5a4efd2e7f476e.png new file mode 100644 index 0000000000000000000000000000000000000000..d3414820504d1187248c5566bf41904c2f85ce32 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_b541dd5a4efd2e7f476e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_d81856892dae7f37a9ef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_d81856892dae7f37a9ef.png new file mode 100644 index 0000000000000000000000000000000000000000..19f81d7d1ca7c35c1bb2908c350577e0afa6f19b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_d81856892dae7f37a9ef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_ed68c4b1954c2b559c72.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_ed68c4b1954c2b559c72.png new file mode 100644 index 0000000000000000000000000000000000000000..e3e0df0dbe75c4791a43c083a61cf5ad21d91804 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_400_ed68c4b1954c2b559c72.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_5100cba967fc4cc1cdf6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_5100cba967fc4cc1cdf6.png new file mode 100644 index 0000000000000000000000000000000000000000..9516d0f5fb2990deb3e5464a17ef1b2f0c9b309a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_5100cba967fc4cc1cdf6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_8049b1e57a48ac722fb3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_8049b1e57a48ac722fb3.png new file mode 100644 index 0000000000000000000000000000000000000000..ca82e35a3aca0ecdea7869c7a12921c2bb3d69b9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_8049b1e57a48ac722fb3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_db3da578241ccb488fb6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_db3da578241ccb488fb6.png new file mode 100644 index 0000000000000000000000000000000000000000..6b967069886640beafab77dacd1e47c2c165c2d7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_db3da578241ccb488fb6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_dea2f3fa8cc40610700c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_dea2f3fa8cc40610700c.png new file mode 100644 index 0000000000000000000000000000000000000000..8702d1894ffed9807516bce0cca2b05ee3c006c3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_408_dea2f3fa8cc40610700c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_144fef68c2f0b44f2e84.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_144fef68c2f0b44f2e84.png new file mode 100644 index 0000000000000000000000000000000000000000..730d36d2689fd412713eb29780780d1b4ff28265 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_144fef68c2f0b44f2e84.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_18d5333919a6d64e6d60.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_18d5333919a6d64e6d60.png new file mode 100644 index 0000000000000000000000000000000000000000..9f20039014f7a68142c1b973692feb5cf78a7306 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_18d5333919a6d64e6d60.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_549de1ccb1807d11a4b8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_549de1ccb1807d11a4b8.png new file mode 100644 index 0000000000000000000000000000000000000000..dbe5feef3864ab52205652c42b7adf8458b0049e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_549de1ccb1807d11a4b8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_86b79bd8a5f65f5b321b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_86b79bd8a5f65f5b321b.png new file mode 100644 index 0000000000000000000000000000000000000000..2bfdba548bcedfbce55485c27ef9b43e654b0258 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_40_86b79bd8a5f65f5b321b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_0fb15379db5c9ff2e0fc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_0fb15379db5c9ff2e0fc.png new file mode 100644 index 0000000000000000000000000000000000000000..2faf554e612d0f499bbb515111d9f329a26289ff Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_0fb15379db5c9ff2e0fc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_33b6f1678b8b6a41d991.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_33b6f1678b8b6a41d991.png new file mode 100644 index 0000000000000000000000000000000000000000..bf61762486e588adb2ddaf3726650a2cc54776d4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_33b6f1678b8b6a41d991.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_cb07e3b163172a97c565.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_cb07e3b163172a97c565.png new file mode 100644 index 0000000000000000000000000000000000000000..8fee744411286d92b4f1814bb2addc457e3eb70c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_cb07e3b163172a97c565.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_d31ae94219028126481b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_d31ae94219028126481b.png new file mode 100644 index 0000000000000000000000000000000000000000..9a3a268e5caa5bd403c6ec57ca4dc968895c2292 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_416_d31ae94219028126481b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_33240f6708272a52eea8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_33240f6708272a52eea8.png new file mode 100644 index 0000000000000000000000000000000000000000..ab1c215ee4d270f0c3dbe046a44f92defb176a28 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_33240f6708272a52eea8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_3e2d62ab8a779b41aad1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_3e2d62ab8a779b41aad1.png new file mode 100644 index 0000000000000000000000000000000000000000..8f80066be842d0f3b4822972a121722f40a6cfa0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_3e2d62ab8a779b41aad1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_85fe6cca04ba628c90f1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_85fe6cca04ba628c90f1.png new file mode 100644 index 0000000000000000000000000000000000000000..588e6517978d14e1de9e5e8983e5340b28c25baa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_85fe6cca04ba628c90f1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_ff3c9534a6bae344d14f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_ff3c9534a6bae344d14f.png new file mode 100644 index 0000000000000000000000000000000000000000..4be20e21bb7afdeead9454923232f36059a0f5d9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_424_ff3c9534a6bae344d14f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_0dfe0505f8d9adcc6ad1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_0dfe0505f8d9adcc6ad1.png new file mode 100644 index 0000000000000000000000000000000000000000..14f6e73438ebc112351e4feae7c19c8f9b3770ec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_0dfe0505f8d9adcc6ad1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_18313983b36cde50dd3b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_18313983b36cde50dd3b.png new file mode 100644 index 0000000000000000000000000000000000000000..e91c42bcc3a08f3acf377b0dab8223c0135225a3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_18313983b36cde50dd3b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_9aaaaf8c3f6ad3ebcd99.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_9aaaaf8c3f6ad3ebcd99.png new file mode 100644 index 0000000000000000000000000000000000000000..f9bf62a64dc2fa765b1a65a1846325b8744084f7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_9aaaaf8c3f6ad3ebcd99.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_c7dc0e4c561f71eb5b3e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_c7dc0e4c561f71eb5b3e.png new file mode 100644 index 0000000000000000000000000000000000000000..6ec50a67b036ba2e6d81f450dd45ef3ee5bfb41f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_432_c7dc0e4c561f71eb5b3e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_12692f8ccbda27de07b9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_12692f8ccbda27de07b9.png new file mode 100644 index 0000000000000000000000000000000000000000..76c33951472c0917dfd823837126ffb03624e89d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_12692f8ccbda27de07b9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_b23e879f7839905b9cab.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_b23e879f7839905b9cab.png new file mode 100644 index 0000000000000000000000000000000000000000..17079f5fb296f44a715023d3fcf51f6f05542a8c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_b23e879f7839905b9cab.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_bf3806856d1001a26a5b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_bf3806856d1001a26a5b.png new file mode 100644 index 0000000000000000000000000000000000000000..51e8fcca691b99eb1929b51a03b3baff96f895a6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_bf3806856d1001a26a5b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_d5960ae197569bb66752.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_d5960ae197569bb66752.png new file mode 100644 index 0000000000000000000000000000000000000000..5abdc27f05bcb84d4971426695602bcbb6cdd0a1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_440_d5960ae197569bb66752.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_9d3f7736aabd23e4c942.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_9d3f7736aabd23e4c942.png new file mode 100644 index 0000000000000000000000000000000000000000..9e4e341820df0575db55457366d10a2e17489a42 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_9d3f7736aabd23e4c942.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_9e5ba8c3ea8c4ba6489b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_9e5ba8c3ea8c4ba6489b.png new file mode 100644 index 0000000000000000000000000000000000000000..ceeefa01ac29d3501602c77ee23471c283216c62 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_9e5ba8c3ea8c4ba6489b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_a781cbefdd5dbd10be81.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_a781cbefdd5dbd10be81.png new file mode 100644 index 0000000000000000000000000000000000000000..815460aae9b235eb0cebdd650ab0c3f9d39d449b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_a781cbefdd5dbd10be81.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_b685a88a96b2c1a8a665.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_b685a88a96b2c1a8a665.png new file mode 100644 index 0000000000000000000000000000000000000000..074eaaf82d96379fc8b8c331cf1d42b1af56ee7f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_448_b685a88a96b2c1a8a665.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_3c82fec3d730872e6a5e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_3c82fec3d730872e6a5e.png new file mode 100644 index 0000000000000000000000000000000000000000..7ea674683e4bb4793e2a89e1199d3bf83354e669 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_3c82fec3d730872e6a5e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_7274f3777b238c5f0628.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_7274f3777b238c5f0628.png new file mode 100644 index 0000000000000000000000000000000000000000..d4e522b7766a0d2ca7d181ee1c80bd6f927b3a5f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_7274f3777b238c5f0628.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_8e44c66b80c9ca0412ba.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_8e44c66b80c9ca0412ba.png new file mode 100644 index 0000000000000000000000000000000000000000..127e21e8b5321d1039d3cad1cfc325d3b2cf876d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_8e44c66b80c9ca0412ba.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_ed31315a0d1d059dae78.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_ed31315a0d1d059dae78.png new file mode 100644 index 0000000000000000000000000000000000000000..15dcfef1669f62751ba65200b797b3666095252b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_456_ed31315a0d1d059dae78.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_66cbaba9a39ca8bfc815.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_66cbaba9a39ca8bfc815.png new file mode 100644 index 0000000000000000000000000000000000000000..5f47f6fe391b5a3f2737b6c69d0eb586a53a8dee Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_66cbaba9a39ca8bfc815.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_97d923b01964cb8ebb37.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_97d923b01964cb8ebb37.png new file mode 100644 index 0000000000000000000000000000000000000000..3b93472214d37c0718f138b1d461e087e2c2d1f5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_97d923b01964cb8ebb37.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_b0bdbc989e686d54eacb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_b0bdbc989e686d54eacb.png new file mode 100644 index 0000000000000000000000000000000000000000..80a8476b52ff9bdc63c652df571529cbbc0567dc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_b0bdbc989e686d54eacb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_b52266a063208eac90d1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_b52266a063208eac90d1.png new file mode 100644 index 0000000000000000000000000000000000000000..441e34851e7c5779abb5263c767d1b5acd0abeed Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_464_b52266a063208eac90d1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_059ba63ee59da583f8b0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_059ba63ee59da583f8b0.png new file mode 100644 index 0000000000000000000000000000000000000000..3460186ce28fcac4c10b89e9c420c9aa724bbff3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_059ba63ee59da583f8b0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_820adba40c6ce80d2f4a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_820adba40c6ce80d2f4a.png new file mode 100644 index 0000000000000000000000000000000000000000..87740de71f9c537de4b7081b1ea807cbcc294b59 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_820adba40c6ce80d2f4a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_ce05e5603e88c594f7a0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_ce05e5603e88c594f7a0.png new file mode 100644 index 0000000000000000000000000000000000000000..28ba93a8df08426eea334bee028d0bec5dec391a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_ce05e5603e88c594f7a0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_cfb31d01ff9d8fcc9935.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_cfb31d01ff9d8fcc9935.png new file mode 100644 index 0000000000000000000000000000000000000000..4e65790f183361981ed78ce9b8e8ce33b3e5d3db Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_472_cfb31d01ff9d8fcc9935.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_a98a50da4795b7ed390e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_a98a50da4795b7ed390e.png new file mode 100644 index 0000000000000000000000000000000000000000..67d8459da9645ab98e3de3bde121b2b230744c15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_a98a50da4795b7ed390e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_be6c1fc77e931ea5eccf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_be6c1fc77e931ea5eccf.png new file mode 100644 index 0000000000000000000000000000000000000000..4e44c41b275dfd487bbb36e60422a840d1b0ec96 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_be6c1fc77e931ea5eccf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_ccca929b7019e723af09.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_ccca929b7019e723af09.png new file mode 100644 index 0000000000000000000000000000000000000000..d9dc413db4984395ad9581649416e7252dff4ec5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_ccca929b7019e723af09.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_cccdf5a911d086c92c1b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_cccdf5a911d086c92c1b.png new file mode 100644 index 0000000000000000000000000000000000000000..53f3fd8f26fcacf99d845b37d5965772918999fd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_480_cccdf5a911d086c92c1b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_1f7f3d27cf4d0d695023.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_1f7f3d27cf4d0d695023.png new file mode 100644 index 0000000000000000000000000000000000000000..4e862abb6e0d473bba08b02ed7aa18f9677ae64f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_1f7f3d27cf4d0d695023.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_446494043ddef0089999.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_446494043ddef0089999.png new file mode 100644 index 0000000000000000000000000000000000000000..0fb32e799a2a0c7b291833950ebeb8de53433e30 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_446494043ddef0089999.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_edd688b52163221b6553.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_edd688b52163221b6553.png new file mode 100644 index 0000000000000000000000000000000000000000..0912ae296dd5d203ce3406d2ec349bb5a3cd8552 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_edd688b52163221b6553.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_f79a2569df6f42330b22.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_f79a2569df6f42330b22.png new file mode 100644 index 0000000000000000000000000000000000000000..d8a7e064e1e4beb09012a8c658b0f4fb0a6d2ea7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_488_f79a2569df6f42330b22.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_0262af6aae483d433db7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_0262af6aae483d433db7.png new file mode 100644 index 0000000000000000000000000000000000000000..330480cd01e0e113e66ab3bf8189c0cd86a0baa4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_0262af6aae483d433db7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_9c214adc0d7f41183109.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_9c214adc0d7f41183109.png new file mode 100644 index 0000000000000000000000000000000000000000..0c23cda4f252da907181c7bdaa6a83891340f077 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_9c214adc0d7f41183109.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_a907fc518966b66deb3c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_a907fc518966b66deb3c.png new file mode 100644 index 0000000000000000000000000000000000000000..c851d16fc3d948fbddb63455387bb11bf52f94dc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_a907fc518966b66deb3c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_d2e4462cea1aed1f2937.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_d2e4462cea1aed1f2937.png new file mode 100644 index 0000000000000000000000000000000000000000..af5b34ffe10a49b0c7912ac3f76ab52be45e58a6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_48_d2e4462cea1aed1f2937.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_2a1d91ee7dad3aca3377.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_2a1d91ee7dad3aca3377.png new file mode 100644 index 0000000000000000000000000000000000000000..1e14802ea2e8eb55f221f807d4b2cc374f5e237f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_2a1d91ee7dad3aca3377.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_37d891b4d6c71e28eca9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_37d891b4d6c71e28eca9.png new file mode 100644 index 0000000000000000000000000000000000000000..8386a61f638a8a3a47347dc021473fd71d552507 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_37d891b4d6c71e28eca9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_68e06148c0b530b292ef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_68e06148c0b530b292ef.png new file mode 100644 index 0000000000000000000000000000000000000000..38255f830504d2c2875b1adf224b0d5eb70a1bc7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_68e06148c0b530b292ef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_ab5da33b4fab6d97badb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_ab5da33b4fab6d97badb.png new file mode 100644 index 0000000000000000000000000000000000000000..13d76433fb12d7107e1c1c298ead4eabc4914901 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_496_ab5da33b4fab6d97badb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_4cd9dbf50122f56d08c9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_4cd9dbf50122f56d08c9.png new file mode 100644 index 0000000000000000000000000000000000000000..59619944a007b32ce6c32d877af45029d8478c69 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_4cd9dbf50122f56d08c9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_8d503921a8e36638c133.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_8d503921a8e36638c133.png new file mode 100644 index 0000000000000000000000000000000000000000..d44c8ffd7f70e7df8111c9e4c4e787787ac4a71c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_8d503921a8e36638c133.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_aab7b24d7b2a7c51475f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_aab7b24d7b2a7c51475f.png new file mode 100644 index 0000000000000000000000000000000000000000..8c60a0b8b4a3a9009d351e870c7150855f1fc654 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_aab7b24d7b2a7c51475f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_f98f85886287d586fb24.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_f98f85886287d586fb24.png new file mode 100644 index 0000000000000000000000000000000000000000..5061a15dae03f64a0914f10b334445069245a4de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_504_f98f85886287d586fb24.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_87b03e3c08874339a548.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_87b03e3c08874339a548.png new file mode 100644 index 0000000000000000000000000000000000000000..0358b54d30ec39e650156cc6fc094ddc2ad17514 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_87b03e3c08874339a548.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_95c88d4ee00e6dceedec.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_95c88d4ee00e6dceedec.png new file mode 100644 index 0000000000000000000000000000000000000000..8dbf083a87c74926f6523ab673e3b1686ebd877a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_95c88d4ee00e6dceedec.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_b6b8325d8fe282e8ee86.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_b6b8325d8fe282e8ee86.png new file mode 100644 index 0000000000000000000000000000000000000000..a1f291b7c2c9ce11ba5835de16bc39984f252cec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_b6b8325d8fe282e8ee86.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_c7af5aea69ad1b787bdd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_c7af5aea69ad1b787bdd.png new file mode 100644 index 0000000000000000000000000000000000000000..529e11054479a8114ecab64a6ab4588dfd4ac6a6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_512_c7af5aea69ad1b787bdd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_2111edf7d29b2a9e9ffa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_2111edf7d29b2a9e9ffa.png new file mode 100644 index 0000000000000000000000000000000000000000..2b6d2deb2f1b3d252fe3b563469795cd5c9332ba Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_2111edf7d29b2a9e9ffa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_3a4cf4d788de75533c67.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_3a4cf4d788de75533c67.png new file mode 100644 index 0000000000000000000000000000000000000000..378a03b40f0f9cdc8f8fcbfdb1b70fb970fd2371 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_3a4cf4d788de75533c67.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_78d0488f99cba790913e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_78d0488f99cba790913e.png new file mode 100644 index 0000000000000000000000000000000000000000..19ccd5cf57d1b45003516b2ff59fe69303a5ccb4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_78d0488f99cba790913e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_b5b4ceac10276cd5b030.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_b5b4ceac10276cd5b030.png new file mode 100644 index 0000000000000000000000000000000000000000..3221e5542a15a0fedee619ee42d590c209de4b89 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_520_b5b4ceac10276cd5b030.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_5348c071dbdd44e6d366.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_5348c071dbdd44e6d366.png new file mode 100644 index 0000000000000000000000000000000000000000..bd29d36aa7ba5bfddc7c524046e3f720be80742c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_5348c071dbdd44e6d366.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_767c54c5dd821b8fd78d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_767c54c5dd821b8fd78d.png new file mode 100644 index 0000000000000000000000000000000000000000..946b2bb1087dadd08cbb32ab7f21f11e702135e3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_767c54c5dd821b8fd78d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_875885eb3355d2fc1a7a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_875885eb3355d2fc1a7a.png new file mode 100644 index 0000000000000000000000000000000000000000..62cf6bff4a2f1d4ecb6fab691040fda968873ce7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_875885eb3355d2fc1a7a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_b7baa092f64c64dba260.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_b7baa092f64c64dba260.png new file mode 100644 index 0000000000000000000000000000000000000000..6302ea0bbf170330dd6897a83c4aca4c6ea8a5a6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_528_b7baa092f64c64dba260.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_08ad374c5434931379b1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_08ad374c5434931379b1.png new file mode 100644 index 0000000000000000000000000000000000000000..bcb93a4291674f8489d58bb8bfa262c0ad25ad5b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_08ad374c5434931379b1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_915b0997964674e19d30.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_915b0997964674e19d30.png new file mode 100644 index 0000000000000000000000000000000000000000..7be863b9e22b83f0d7f6f324132b0a683ca9ec8e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_915b0997964674e19d30.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_a8e5af09990ec6f7b046.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_a8e5af09990ec6f7b046.png new file mode 100644 index 0000000000000000000000000000000000000000..4a30571df027a189be145d7163e539213608ab5c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_a8e5af09990ec6f7b046.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_e6b2848e4a4a5a1fa3b2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_e6b2848e4a4a5a1fa3b2.png new file mode 100644 index 0000000000000000000000000000000000000000..4411a1e2ba2f7000a9d90fb26b5f7b8b544380ff Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_536_e6b2848e4a4a5a1fa3b2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_b99da3c0fcb7ededf7b0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_b99da3c0fcb7ededf7b0.png new file mode 100644 index 0000000000000000000000000000000000000000..01bb02ad0ce463afeec68d1a98d378690434c4a4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_b99da3c0fcb7ededf7b0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_d527e1f00fb003991478.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_d527e1f00fb003991478.png new file mode 100644 index 0000000000000000000000000000000000000000..12548bd0130e0c2f856b578fe7b01e350b3caa4b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_d527e1f00fb003991478.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_ec40722860c852345c23.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_ec40722860c852345c23.png new file mode 100644 index 0000000000000000000000000000000000000000..a257f8b85bed93e73020eb82e7b0b0fc96f4025f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_ec40722860c852345c23.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_f145c478050d42496e6e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_f145c478050d42496e6e.png new file mode 100644 index 0000000000000000000000000000000000000000..0f58f927e3221f934fe89c37b83c71c772a4bef8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_544_f145c478050d42496e6e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_257d3c0b8f2426466221.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_257d3c0b8f2426466221.png new file mode 100644 index 0000000000000000000000000000000000000000..bc29992e28e5ba9ad9ca9077d4c717e13e4e6a20 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_257d3c0b8f2426466221.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_41181c5e1fbbaa3688f6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_41181c5e1fbbaa3688f6.png new file mode 100644 index 0000000000000000000000000000000000000000..cf27682e72a314a6d8f98ffe9b8e7010ef7732a5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_41181c5e1fbbaa3688f6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_8812f45a50bdc79a8517.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_8812f45a50bdc79a8517.png new file mode 100644 index 0000000000000000000000000000000000000000..e441e31ec99a57b72bb50c8107672d19ea5d5fe3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_8812f45a50bdc79a8517.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_8c78fd5084c9eef35e50.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_8c78fd5084c9eef35e50.png new file mode 100644 index 0000000000000000000000000000000000000000..1ff4a054a8d14ca30b8f2bc23e78aad6394fac2b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_552_8c78fd5084c9eef35e50.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_8a39c93dbe4043d18b4f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_8a39c93dbe4043d18b4f.png new file mode 100644 index 0000000000000000000000000000000000000000..163ebac81d768fb11dfa2871b7e2be88b847192f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_8a39c93dbe4043d18b4f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_b956f5c40cffd0dcba98.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_b956f5c40cffd0dcba98.png new file mode 100644 index 0000000000000000000000000000000000000000..88e518d6897e45e971050005a7aae218136b7c73 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_b956f5c40cffd0dcba98.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_bbef43cf934160e04cfd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_bbef43cf934160e04cfd.png new file mode 100644 index 0000000000000000000000000000000000000000..91af2ad46ffa7e5ae4eb35d2e298ef290135aef6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_bbef43cf934160e04cfd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_f76cf11600d692be2502.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_f76cf11600d692be2502.png new file mode 100644 index 0000000000000000000000000000000000000000..dd26677156de2c824629b781944e84a946658c8c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_560_f76cf11600d692be2502.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_04d62c7653c5d162a2e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_04d62c7653c5d162a2e3.png new file mode 100644 index 0000000000000000000000000000000000000000..372e59ff9c627cc95557261e758a8286d27ada61 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_04d62c7653c5d162a2e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_68596aff73764ec36f54.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_68596aff73764ec36f54.png new file mode 100644 index 0000000000000000000000000000000000000000..bf42dfa8c721530f3251682ada6a03991ac6abec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_68596aff73764ec36f54.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_dba4f198a349b1343cdd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_dba4f198a349b1343cdd.png new file mode 100644 index 0000000000000000000000000000000000000000..20dca625401f049f479b6303851c1f00cf9df10b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_dba4f198a349b1343cdd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_e297ab1a0ce4b9855bb5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_e297ab1a0ce4b9855bb5.png new file mode 100644 index 0000000000000000000000000000000000000000..05a630ac6a31b0cd188cf22054a0cd74975d1cc5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_568_e297ab1a0ce4b9855bb5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_12bc42303d2e7628fd3f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_12bc42303d2e7628fd3f.png new file mode 100644 index 0000000000000000000000000000000000000000..94ca353aa40e20bc7feed5c3783c13c9ca51e999 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_12bc42303d2e7628fd3f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_1536eefdcfae05b945b9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_1536eefdcfae05b945b9.png new file mode 100644 index 0000000000000000000000000000000000000000..88db38601b24740ea2843c8005cbff485f9cb195 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_1536eefdcfae05b945b9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_664030382ee576a5c9b2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_664030382ee576a5c9b2.png new file mode 100644 index 0000000000000000000000000000000000000000..273c776be0663d34b1f61754202a4da88241c2f0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_664030382ee576a5c9b2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_ef4ee0a1a460a56f4751.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_ef4ee0a1a460a56f4751.png new file mode 100644 index 0000000000000000000000000000000000000000..651a3b2cf33bcf8feb517e5539a05a9728ae2d67 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_56_ef4ee0a1a460a56f4751.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_0fa5d5fa63f3526989d2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_0fa5d5fa63f3526989d2.png new file mode 100644 index 0000000000000000000000000000000000000000..c589c3a6732bd172f101c99fe22a408ed42da755 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_0fa5d5fa63f3526989d2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_3b7a05a66d1f89e67bb8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_3b7a05a66d1f89e67bb8.png new file mode 100644 index 0000000000000000000000000000000000000000..b8dc2fb35e0e5a0009c255106898322966536762 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_3b7a05a66d1f89e67bb8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_b1cd8c64caeaede72534.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_b1cd8c64caeaede72534.png new file mode 100644 index 0000000000000000000000000000000000000000..90470d792b22aca17b2264419666ae1647c57238 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_b1cd8c64caeaede72534.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_b8f228173e9910efda0e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_b8f228173e9910efda0e.png new file mode 100644 index 0000000000000000000000000000000000000000..efef92fa728e31ebc805505eb09b5a1781ca3866 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_576_b8f228173e9910efda0e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_29997c31305c3912cd04.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_29997c31305c3912cd04.png new file mode 100644 index 0000000000000000000000000000000000000000..671472168bccbe6a6c293a7fdf34e286e882ebaa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_29997c31305c3912cd04.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_434c1624a967a3f3a962.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_434c1624a967a3f3a962.png new file mode 100644 index 0000000000000000000000000000000000000000..ae3f70f2a867451d322f5398b698a23afebf002b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_434c1624a967a3f3a962.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_b49048e90ace4b32dca7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_b49048e90ace4b32dca7.png new file mode 100644 index 0000000000000000000000000000000000000000..ddbf2f4fef36c677aaccbfdc666e52f2e776067b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_b49048e90ace4b32dca7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_fea9e895a6bad4c69a30.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_fea9e895a6bad4c69a30.png new file mode 100644 index 0000000000000000000000000000000000000000..9315a07322889f0b52439f6bc6739df602491564 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_584_fea9e895a6bad4c69a30.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_0e50e9bfb15ce39c112d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_0e50e9bfb15ce39c112d.png new file mode 100644 index 0000000000000000000000000000000000000000..86ddd2b72ae031092c1dfb45bba5ff63c42af659 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_0e50e9bfb15ce39c112d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_3924670710a9e29d06d3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_3924670710a9e29d06d3.png new file mode 100644 index 0000000000000000000000000000000000000000..a4590dc3853228c7ec31e52a012eb4e35b9c7bd0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_3924670710a9e29d06d3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_9a03f64388706e1447b6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_9a03f64388706e1447b6.png new file mode 100644 index 0000000000000000000000000000000000000000..e02d8f38c2899622560c1e65aa34ff07675e7014 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_9a03f64388706e1447b6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_cd080a9de17f2873d239.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_cd080a9de17f2873d239.png new file mode 100644 index 0000000000000000000000000000000000000000..f811ed1df1381f68531a40a84b4158fc0b8146ec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_592_cd080a9de17f2873d239.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_1d97a6cab0fe06041ca1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_1d97a6cab0fe06041ca1.png new file mode 100644 index 0000000000000000000000000000000000000000..44e5b2b5801a8a595e5ec641f54a1318630bf1af Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_1d97a6cab0fe06041ca1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_4dddb5352b02ea5893dc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_4dddb5352b02ea5893dc.png new file mode 100644 index 0000000000000000000000000000000000000000..6de71434d7fce22de8f2a88f9454a89c1b877949 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_4dddb5352b02ea5893dc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_8300fd15af7363a95498.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_8300fd15af7363a95498.png new file mode 100644 index 0000000000000000000000000000000000000000..5f9e6cea314659b929e26d81089a39f201581ea3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_8300fd15af7363a95498.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_917bd69bfc0758359482.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_917bd69bfc0758359482.png new file mode 100644 index 0000000000000000000000000000000000000000..50d7bcc9ff65f7f0f4bf43d9091f90d6ea81c66b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_600_917bd69bfc0758359482.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_0f027a7a2f974076b6e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_0f027a7a2f974076b6e9.png new file mode 100644 index 0000000000000000000000000000000000000000..d754b02bfc6059fff5ef40b365df31bdd50b4586 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_0f027a7a2f974076b6e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_2104758725886dd01ccd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_2104758725886dd01ccd.png new file mode 100644 index 0000000000000000000000000000000000000000..7aa7ff716958b856de88f8b3c3f6a400ffc85d49 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_2104758725886dd01ccd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_6863da4ca0ed68a9855f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_6863da4ca0ed68a9855f.png new file mode 100644 index 0000000000000000000000000000000000000000..f377b5a4edcb0a12f13d3aaa5ae5f16d16f0421a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_6863da4ca0ed68a9855f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_e93612b41299320c0daf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_e93612b41299320c0daf.png new file mode 100644 index 0000000000000000000000000000000000000000..3714a9c739cf876d69d9771a664aad10917bdfdb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_608_e93612b41299320c0daf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_3cf079583c2c3172a1af.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_3cf079583c2c3172a1af.png new file mode 100644 index 0000000000000000000000000000000000000000..6a5b7a073c3efcad65f8166b45d426fcdad6f14b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_3cf079583c2c3172a1af.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_7dcb623099bcf2fb9044.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_7dcb623099bcf2fb9044.png new file mode 100644 index 0000000000000000000000000000000000000000..bd297f259b9686cc3428ae5a504e23d13868743e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_7dcb623099bcf2fb9044.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_a89cec9bd984af45d04e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_a89cec9bd984af45d04e.png new file mode 100644 index 0000000000000000000000000000000000000000..48ba123afdeeb6ec25a59d0ff8af917734914389 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_a89cec9bd984af45d04e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_b8d74a65b6fc0e8293ef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_b8d74a65b6fc0e8293ef.png new file mode 100644 index 0000000000000000000000000000000000000000..02bfb0aab17ff901374edb39b53547c1f824378d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_616_b8d74a65b6fc0e8293ef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_5194e21fa06d24dfa3a2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_5194e21fa06d24dfa3a2.png new file mode 100644 index 0000000000000000000000000000000000000000..a0d544e10cee9ad7a5e847b972f040012f8b3f82 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_5194e21fa06d24dfa3a2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_7988a620d724c156773b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_7988a620d724c156773b.png new file mode 100644 index 0000000000000000000000000000000000000000..0fcdbc8b4630db3c297dd9b53b5ad10aec317f2a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_7988a620d724c156773b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_870de5f70c0e3c35f00a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_870de5f70c0e3c35f00a.png new file mode 100644 index 0000000000000000000000000000000000000000..30dff688bfc56c342e1f85af6250a6c46fe7c088 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_870de5f70c0e3c35f00a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_9cd574bf60106cbeb7f7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_9cd574bf60106cbeb7f7.png new file mode 100644 index 0000000000000000000000000000000000000000..175e3fedd4d2fd92f3ec59765a73b53e876db825 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_624_9cd574bf60106cbeb7f7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_90d4a1e76489230b3d0e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_90d4a1e76489230b3d0e.png new file mode 100644 index 0000000000000000000000000000000000000000..859c2415388b401512301801d339ef2910b5e984 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_90d4a1e76489230b3d0e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_9db58ceffde2ac6026b2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_9db58ceffde2ac6026b2.png new file mode 100644 index 0000000000000000000000000000000000000000..ddf874d05dc3d0c22fd34f46ca0dd2baa932383c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_9db58ceffde2ac6026b2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_cbe1621c20b110b72424.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_cbe1621c20b110b72424.png new file mode 100644 index 0000000000000000000000000000000000000000..00f63e5eb9de6eccb684a23b260bc91d1d4798c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_cbe1621c20b110b72424.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_e3edcf43076ac8fbe703.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_e3edcf43076ac8fbe703.png new file mode 100644 index 0000000000000000000000000000000000000000..ad5bb165168425c64a54b65681e272ceb9fa06f3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_632_e3edcf43076ac8fbe703.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_926cd9184da41f18f64c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_926cd9184da41f18f64c.png new file mode 100644 index 0000000000000000000000000000000000000000..fa409e9741b2e5a112728e972967af1fbc19a6f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_926cd9184da41f18f64c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_988f0fde8c0b0bda7e16.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_988f0fde8c0b0bda7e16.png new file mode 100644 index 0000000000000000000000000000000000000000..7c34503c98a2f99e89a5183137f476055f17bbe5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_988f0fde8c0b0bda7e16.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_b0840e49c29d03b310d6.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_b0840e49c29d03b310d6.png new file mode 100644 index 0000000000000000000000000000000000000000..04c0280b3a415c746d61d83e3a3dd2ab29c54d6c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_b0840e49c29d03b310d6.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_cf8925a87c42c9b29e53.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_cf8925a87c42c9b29e53.png new file mode 100644 index 0000000000000000000000000000000000000000..3be134839037c3a76592c8d70f4b5802072e3f82 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_640_cf8925a87c42c9b29e53.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_55287d9a3ca04b88ba22.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_55287d9a3ca04b88ba22.png new file mode 100644 index 0000000000000000000000000000000000000000..a0a32c736be2e9896b1ed5c48f84651e3ea35d04 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_55287d9a3ca04b88ba22.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_5b836be338a558b667f9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_5b836be338a558b667f9.png new file mode 100644 index 0000000000000000000000000000000000000000..d9b4527323e55fd5f54131078cebce3dd6aef2fe Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_5b836be338a558b667f9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_728dd91334abb53b86d0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_728dd91334abb53b86d0.png new file mode 100644 index 0000000000000000000000000000000000000000..f556004b50d6ae1b17bbe120813981e49a83008e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_728dd91334abb53b86d0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_7c2b88efee87b1a3d3bc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_7c2b88efee87b1a3d3bc.png new file mode 100644 index 0000000000000000000000000000000000000000..75b09274665d02ea87a5a802ff8a5e2ac839f0f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_648_7c2b88efee87b1a3d3bc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_2537a2652dff56481399.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_2537a2652dff56481399.png new file mode 100644 index 0000000000000000000000000000000000000000..6705bf1ee1d58d409b40fc827b5420661e9e9033 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_2537a2652dff56481399.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_3e87611fd915ec96e580.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_3e87611fd915ec96e580.png new file mode 100644 index 0000000000000000000000000000000000000000..eff0a80e3b823af0f615cf73ed51e8d9f1df055e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_3e87611fd915ec96e580.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_75bb28ff057271dedef7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_75bb28ff057271dedef7.png new file mode 100644 index 0000000000000000000000000000000000000000..99e3790109522189a3ad6c7c43d76ad211f37db6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_75bb28ff057271dedef7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_9bd0267d729986b95a5d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_9bd0267d729986b95a5d.png new file mode 100644 index 0000000000000000000000000000000000000000..dfc387d90a95232c4ea837bbb6d960b04a8582ae Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_64_9bd0267d729986b95a5d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_09858d6b1f00f78cae5c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_09858d6b1f00f78cae5c.png new file mode 100644 index 0000000000000000000000000000000000000000..4d50db2eec07003166c105364a1bdb4a32949c12 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_09858d6b1f00f78cae5c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_44ad3c8fcf80a92c74c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_44ad3c8fcf80a92c74c0.png new file mode 100644 index 0000000000000000000000000000000000000000..75ae7084b566d4e3804adadabef6d1213d8a5dce Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_44ad3c8fcf80a92c74c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_c5399b2a25bca627fffe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_c5399b2a25bca627fffe.png new file mode 100644 index 0000000000000000000000000000000000000000..71eb4ad151b86578c9f6994c61bc1c1679888be3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_c5399b2a25bca627fffe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_c7768aa4e07faefe0067.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_c7768aa4e07faefe0067.png new file mode 100644 index 0000000000000000000000000000000000000000..506a1ff228d06b27aee8a679bdddc8077f968418 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_656_c7768aa4e07faefe0067.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_092cf810659ff0a3ad9b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_092cf810659ff0a3ad9b.png new file mode 100644 index 0000000000000000000000000000000000000000..4ec27839443652a4ed87360f42af56c51c0a9612 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_092cf810659ff0a3ad9b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_158f555f9864c3964f75.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_158f555f9864c3964f75.png new file mode 100644 index 0000000000000000000000000000000000000000..29bdec964b0ba404f5ecccdc383b50c5f6dc1e6c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_158f555f9864c3964f75.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_6d9e47d176e82143fca1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_6d9e47d176e82143fca1.png new file mode 100644 index 0000000000000000000000000000000000000000..3ee4cdb535ab9c0a90c15f82ec99f4b2e7dc407b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_6d9e47d176e82143fca1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_95c67446531a911a320b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_95c67446531a911a320b.png new file mode 100644 index 0000000000000000000000000000000000000000..49877e0ae4da1bc3b2b8d3c654682bfc0c96ec4c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_664_95c67446531a911a320b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_0a35aec0325a8b5931bd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_0a35aec0325a8b5931bd.png new file mode 100644 index 0000000000000000000000000000000000000000..6b9a8c150b16fa78bebc3bd33a0b590ab4e09aa9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_0a35aec0325a8b5931bd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_640323be00d22421969d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_640323be00d22421969d.png new file mode 100644 index 0000000000000000000000000000000000000000..018af56b9e726dc651a8f85acd76250602c6b87c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_640323be00d22421969d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_e6e648fc0801971ed27f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_e6e648fc0801971ed27f.png new file mode 100644 index 0000000000000000000000000000000000000000..ff7ee79f0b14edfdbff32983c7c156e2bdcc4eaf Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_e6e648fc0801971ed27f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_e7cccd2e24c981238fe9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_e7cccd2e24c981238fe9.png new file mode 100644 index 0000000000000000000000000000000000000000..0a2fe944763f237c9ef6df38027c67845dbab116 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_672_e7cccd2e24c981238fe9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_5296d50b33663e449a0e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_5296d50b33663e449a0e.png new file mode 100644 index 0000000000000000000000000000000000000000..0ed9418f2d8ab75dfe223f650ad631a0460b40ea Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_5296d50b33663e449a0e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_590049620b94f7f699ff.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_590049620b94f7f699ff.png new file mode 100644 index 0000000000000000000000000000000000000000..51aa2ae2d53bfbad2f0cd5e603a92b3b4de4e0c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_590049620b94f7f699ff.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_72ef33de170d84e10fda.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_72ef33de170d84e10fda.png new file mode 100644 index 0000000000000000000000000000000000000000..c41c0b41bdef3e8513ba9a66b93ae80449d00385 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_72ef33de170d84e10fda.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_aa194dccdb4cde13bf28.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_aa194dccdb4cde13bf28.png new file mode 100644 index 0000000000000000000000000000000000000000..3508fb6ea7c963794a457d93025dc128d4290bfa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_680_aa194dccdb4cde13bf28.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_1987fb0f39545442f7d1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_1987fb0f39545442f7d1.png new file mode 100644 index 0000000000000000000000000000000000000000..da73d30f85a56cd31da9e0323279cf6d87bba388 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_1987fb0f39545442f7d1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_59f2ed6f733f8149780a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_59f2ed6f733f8149780a.png new file mode 100644 index 0000000000000000000000000000000000000000..1b859a5e7785e9a925484baf155edbc07e7b7c39 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_59f2ed6f733f8149780a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_9f4c23b3d6799b0996b4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_9f4c23b3d6799b0996b4.png new file mode 100644 index 0000000000000000000000000000000000000000..5f718398a33da562be2648eee6d72fab7335b4f3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_9f4c23b3d6799b0996b4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_dc7a2400adb3ddfda5f1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_dc7a2400adb3ddfda5f1.png new file mode 100644 index 0000000000000000000000000000000000000000..1255c300963e0a127e7c0968ae4415a897c95875 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_688_dc7a2400adb3ddfda5f1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_597ae9e605560c042f4a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_597ae9e605560c042f4a.png new file mode 100644 index 0000000000000000000000000000000000000000..c20e71009fe0b0f86ced3c2d2274e32669ee049f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_597ae9e605560c042f4a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_72197a206eb0dae2a9b9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_72197a206eb0dae2a9b9.png new file mode 100644 index 0000000000000000000000000000000000000000..216d7d9f6294666087a432742644462f9fad3811 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_72197a206eb0dae2a9b9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_b61dd12befa2c9cd68d2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_b61dd12befa2c9cd68d2.png new file mode 100644 index 0000000000000000000000000000000000000000..dee3dbb713eca0c2b4679c74921023880158b54c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_b61dd12befa2c9cd68d2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_bf57326caaf8c0b6ad64.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_bf57326caaf8c0b6ad64.png new file mode 100644 index 0000000000000000000000000000000000000000..1ca4b22931d06a2ad020ece0728eed3c14a54a05 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_696_bf57326caaf8c0b6ad64.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_072c318a44d7c7157563.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_072c318a44d7c7157563.png new file mode 100644 index 0000000000000000000000000000000000000000..2c34ee1688b20068decd0d22f62c8cbb69a8f119 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_072c318a44d7c7157563.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_81266f0e45c6c5305caa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_81266f0e45c6c5305caa.png new file mode 100644 index 0000000000000000000000000000000000000000..c48b9ecad6e3586f0e07424057c0c374b4da6343 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_81266f0e45c6c5305caa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_c190c2a29844123428a8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_c190c2a29844123428a8.png new file mode 100644 index 0000000000000000000000000000000000000000..d99371c2eacf005ec0c1894ff3427fd78727ee0e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_c190c2a29844123428a8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_ff8ed0b57b72882d609c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_ff8ed0b57b72882d609c.png new file mode 100644 index 0000000000000000000000000000000000000000..9b782e1919da254840762878415c499c26b778f6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_704_ff8ed0b57b72882d609c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_6ba33f9b1438e729f3e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_6ba33f9b1438e729f3e3.png new file mode 100644 index 0000000000000000000000000000000000000000..60701af3d0ae4a34992e5ef59bd326b5f572d5be Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_6ba33f9b1438e729f3e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_7fff33d2cf3b6bfe2855.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_7fff33d2cf3b6bfe2855.png new file mode 100644 index 0000000000000000000000000000000000000000..8c59d606bfde834c6bee50957d50f5a50a44030d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_7fff33d2cf3b6bfe2855.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_a41798bd5ba67711765f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_a41798bd5ba67711765f.png new file mode 100644 index 0000000000000000000000000000000000000000..f5b1476dee45b61503cf71dfe8473555aab05f1d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_a41798bd5ba67711765f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_e66ad705dd270fae09fa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_e66ad705dd270fae09fa.png new file mode 100644 index 0000000000000000000000000000000000000000..9dfee77411cbd37d63aff7bba36bb9b8e92bdc87 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_712_e66ad705dd270fae09fa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_18dae1edaa995fefc02c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_18dae1edaa995fefc02c.png new file mode 100644 index 0000000000000000000000000000000000000000..d2952372011d7cb4580af75bf5e506182f53b47a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_18dae1edaa995fefc02c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_3ac906f6178bc7cc5030.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_3ac906f6178bc7cc5030.png new file mode 100644 index 0000000000000000000000000000000000000000..15666dc76b2dc7d3e2dfc2767ff4c01a4774986f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_3ac906f6178bc7cc5030.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_a76739d06259d72923e3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_a76739d06259d72923e3.png new file mode 100644 index 0000000000000000000000000000000000000000..90221ba6c6c42a1e2f3bb02a82ceff011da29192 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_a76739d06259d72923e3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_f473f15d1fa6dae04183.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_f473f15d1fa6dae04183.png new file mode 100644 index 0000000000000000000000000000000000000000..f7c34e6c2e9f02d73281bb23c87cc5ff8f3157a9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_720_f473f15d1fa6dae04183.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_6d3391d06d6901eb149d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_6d3391d06d6901eb149d.png new file mode 100644 index 0000000000000000000000000000000000000000..fda1dedeb18922e4d0641240ff4066dbf3da2180 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_6d3391d06d6901eb149d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_7db45a659309d31efe67.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_7db45a659309d31efe67.png new file mode 100644 index 0000000000000000000000000000000000000000..138e3715779bdd3248625fb16da477510372e9aa Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_7db45a659309d31efe67.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_85f42fd5b9d94ea9077f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_85f42fd5b9d94ea9077f.png new file mode 100644 index 0000000000000000000000000000000000000000..39bdc58f16858fceeee34792bed46e0478ac5e35 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_85f42fd5b9d94ea9077f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_b82c35a5387d49e858c0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_b82c35a5387d49e858c0.png new file mode 100644 index 0000000000000000000000000000000000000000..594bc9b4da31bb7b72a0c9eff48b278360154bdb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_728_b82c35a5387d49e858c0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_8f41b5c8c4403cc49f21.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_8f41b5c8c4403cc49f21.png new file mode 100644 index 0000000000000000000000000000000000000000..c10bb0f58331c25dfdae04bda9408d8ad2354a89 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_8f41b5c8c4403cc49f21.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_8f7cadaa97753cdcfd66.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_8f7cadaa97753cdcfd66.png new file mode 100644 index 0000000000000000000000000000000000000000..cf6565262f2101ab7ccf22b79b8f71346c20e266 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_8f7cadaa97753cdcfd66.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_9b536587af12e637b86b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_9b536587af12e637b86b.png new file mode 100644 index 0000000000000000000000000000000000000000..d7ce4392279f384962c61740fd980e96532dcadd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_9b536587af12e637b86b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_c4f043b99e28a96d715b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_c4f043b99e28a96d715b.png new file mode 100644 index 0000000000000000000000000000000000000000..a70759282cfad78071decd28962d7ff3dc067aba Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_72_c4f043b99e28a96d715b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_ac4e29b11ce1d305f530.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_ac4e29b11ce1d305f530.png new file mode 100644 index 0000000000000000000000000000000000000000..bc51ef7ca24809725e46d6c5f294d34b930f04a7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_ac4e29b11ce1d305f530.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_d6436e676d16768a25aa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_d6436e676d16768a25aa.png new file mode 100644 index 0000000000000000000000000000000000000000..d247870c7e566d466209d98554a6bbc7a54d62da Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_d6436e676d16768a25aa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_f0caac32194909cb1981.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_f0caac32194909cb1981.png new file mode 100644 index 0000000000000000000000000000000000000000..837e52f2e897e0e401e494f35979cb702c727044 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_f0caac32194909cb1981.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_fae683dc9806394cb2e9.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_fae683dc9806394cb2e9.png new file mode 100644 index 0000000000000000000000000000000000000000..030ee3f7f94f9f46cce65f87f46bfb4b2999fea2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_736_fae683dc9806394cb2e9.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_24c539112b53ecad6996.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_24c539112b53ecad6996.png new file mode 100644 index 0000000000000000000000000000000000000000..00b0a6546367517bf447b67cd99284269515d375 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_24c539112b53ecad6996.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_54915f91d21e93f22f28.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_54915f91d21e93f22f28.png new file mode 100644 index 0000000000000000000000000000000000000000..ba8161ac3d4608c93a18948eacdbfa00818cfb60 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_54915f91d21e93f22f28.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_835fc7ba76fe5bdd463b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_835fc7ba76fe5bdd463b.png new file mode 100644 index 0000000000000000000000000000000000000000..07f1d0a2a5c83af7ced05fab37d57afaf16d66a2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_835fc7ba76fe5bdd463b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_88c24683fdcc6de68876.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_88c24683fdcc6de68876.png new file mode 100644 index 0000000000000000000000000000000000000000..d075908e8e4cd7f17334503691f888def73793f2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_744_88c24683fdcc6de68876.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_95d256cb61eaa36920d3.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_95d256cb61eaa36920d3.png new file mode 100644 index 0000000000000000000000000000000000000000..f75367a18aaab7ce8c31d67b18d3c3ea7026073f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_95d256cb61eaa36920d3.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_aa82701d58c5e4be3efd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_aa82701d58c5e4be3efd.png new file mode 100644 index 0000000000000000000000000000000000000000..74816da9fa93e30036ad089119174d8e9584da65 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_aa82701d58c5e4be3efd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_e8178d5464133df3775d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_e8178d5464133df3775d.png new file mode 100644 index 0000000000000000000000000000000000000000..9c5220c5a80856da73d1e51b7f994b2ff68739df Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_e8178d5464133df3775d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_ec4a84736de32ae6ae3c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_ec4a84736de32ae6ae3c.png new file mode 100644 index 0000000000000000000000000000000000000000..74fefff4953cf5242de00e210be8898a883fb486 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_752_ec4a84736de32ae6ae3c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_8c7815ce3a6257b58e00.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_8c7815ce3a6257b58e00.png new file mode 100644 index 0000000000000000000000000000000000000000..f41bf15b154fd87fff24233d250c6642cddb84af Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_8c7815ce3a6257b58e00.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_a1a1c2ba06a8ef07f019.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_a1a1c2ba06a8ef07f019.png new file mode 100644 index 0000000000000000000000000000000000000000..a331f2e8240a5f5cdeb11da420bdb8756d544fc8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_a1a1c2ba06a8ef07f019.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_cf617b6b91a851ec265b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_cf617b6b91a851ec265b.png new file mode 100644 index 0000000000000000000000000000000000000000..52e86996a7a6e04898ec73723571a00097aeb65e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_cf617b6b91a851ec265b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_fb0a52e92cd7ee579cca.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_fb0a52e92cd7ee579cca.png new file mode 100644 index 0000000000000000000000000000000000000000..c2fa7eca2af6ac6f824be3c957b0762962f0f47a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_760_fb0a52e92cd7ee579cca.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_282437d359e52d34f4f8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_282437d359e52d34f4f8.png new file mode 100644 index 0000000000000000000000000000000000000000..fc91bb8dded4f7a08260cf61bd5e80f095960b15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_282437d359e52d34f4f8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_373eddb1acba3661e232.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_373eddb1acba3661e232.png new file mode 100644 index 0000000000000000000000000000000000000000..acd010fe1c3b11e428a22ecd90d3159ecf1ad0ff Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_373eddb1acba3661e232.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_693e8b6430b9351126e2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_693e8b6430b9351126e2.png new file mode 100644 index 0000000000000000000000000000000000000000..77141dfaf92e442b44f1197537ebe81f3d4289a1 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_693e8b6430b9351126e2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_917890c98c8478bc01dd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_917890c98c8478bc01dd.png new file mode 100644 index 0000000000000000000000000000000000000000..aa89308c97fba5f3d45b9c8d67b66cf31b35e456 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_768_917890c98c8478bc01dd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_06b4410d2195a28ee2e4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_06b4410d2195a28ee2e4.png new file mode 100644 index 0000000000000000000000000000000000000000..f9aa1027c56f3bb3fd55697a48cc51e74ba72a46 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_06b4410d2195a28ee2e4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_2aac3dd33ac8c1b799cb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_2aac3dd33ac8c1b799cb.png new file mode 100644 index 0000000000000000000000000000000000000000..3ec55f07233bb6b582acd0f2ba74eaa5d3fa16a0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_2aac3dd33ac8c1b799cb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_938b1a9310105c2c30a8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_938b1a9310105c2c30a8.png new file mode 100644 index 0000000000000000000000000000000000000000..09a495e57430aab8f988f50659ce4d61893dc7a3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_938b1a9310105c2c30a8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_eb54d867993cb09e2b53.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_eb54d867993cb09e2b53.png new file mode 100644 index 0000000000000000000000000000000000000000..fed43049b08d325c092136573c1a93b32971b0e9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_776_eb54d867993cb09e2b53.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_07f7d7557663601508eb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_07f7d7557663601508eb.png new file mode 100644 index 0000000000000000000000000000000000000000..cdd125c86e59fae2b80bdf6003d5aa0ac25bf610 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_07f7d7557663601508eb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_0fa68e4c56abe4cd2e0d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_0fa68e4c56abe4cd2e0d.png new file mode 100644 index 0000000000000000000000000000000000000000..8c81a5cdc05696110db612712fa0e2f9f6786a8b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_0fa68e4c56abe4cd2e0d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_489eed56c83ee92b8e9c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_489eed56c83ee92b8e9c.png new file mode 100644 index 0000000000000000000000000000000000000000..7a6e7a5639c8264d0935f4afb8e75fa7a6fd896d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_489eed56c83ee92b8e9c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_731753b83553f4858c73.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_731753b83553f4858c73.png new file mode 100644 index 0000000000000000000000000000000000000000..0861591eb85bfdf7f819a6aedaca55ef8e7ce5a0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_784_731753b83553f4858c73.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_2db0a6b3dbfebedf2d8e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_2db0a6b3dbfebedf2d8e.png new file mode 100644 index 0000000000000000000000000000000000000000..3f19dfb2a645931410b5ab5c6e091870540f8905 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_2db0a6b3dbfebedf2d8e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_3f75331b78a95b5b40b7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_3f75331b78a95b5b40b7.png new file mode 100644 index 0000000000000000000000000000000000000000..3aeaafaa4931af83a24f819c0af9b8a1f53ffaa0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_3f75331b78a95b5b40b7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_e061fd9e41fdfb8c934a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_e061fd9e41fdfb8c934a.png new file mode 100644 index 0000000000000000000000000000000000000000..6c0737078cc295013311aba495dcaa8aad6deda2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_e061fd9e41fdfb8c934a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_eac5499ee475cd33206f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_eac5499ee475cd33206f.png new file mode 100644 index 0000000000000000000000000000000000000000..1a8bd53c9648bc28dbcd2d68070dfcd87f964715 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_792_eac5499ee475cd33206f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_2ed775ff9a30d99a0a43.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_2ed775ff9a30d99a0a43.png new file mode 100644 index 0000000000000000000000000000000000000000..fe64b4a946df380db497a4843b778ef52e630ee5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_2ed775ff9a30d99a0a43.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_31cd8fae17813d7d0e65.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_31cd8fae17813d7d0e65.png new file mode 100644 index 0000000000000000000000000000000000000000..fb0997cad3d10e81b7a4235efaede815ad054a50 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_31cd8fae17813d7d0e65.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_3d6397867b321dddf262.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_3d6397867b321dddf262.png new file mode 100644 index 0000000000000000000000000000000000000000..328a405fcc1dd73e28c7c9ed225635e8d5117c3b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_3d6397867b321dddf262.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_d905700551ce75840705.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_d905700551ce75840705.png new file mode 100644 index 0000000000000000000000000000000000000000..fa7a4c500fea347bf75645412c27c34063782838 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_800_d905700551ce75840705.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_198cfae2d5879abcba46.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_198cfae2d5879abcba46.png new file mode 100644 index 0000000000000000000000000000000000000000..19fce45eec9795ce9df9c12c43fb4115cd088fdb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_198cfae2d5879abcba46.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_7aaebfffee54696e50f8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_7aaebfffee54696e50f8.png new file mode 100644 index 0000000000000000000000000000000000000000..cf301b89c6ade91ce83a54aabc385bf05c9e2056 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_7aaebfffee54696e50f8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_b053e35f6e9aac4bea20.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_b053e35f6e9aac4bea20.png new file mode 100644 index 0000000000000000000000000000000000000000..5033c25f83df578765c0bdb3c0b7dfc1125e37b4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_b053e35f6e9aac4bea20.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_f9c3bd83a1bb11eb4cff.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_f9c3bd83a1bb11eb4cff.png new file mode 100644 index 0000000000000000000000000000000000000000..eb79ddcaf19c38881bf0a9707c414a8dd0a83e24 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_808_f9c3bd83a1bb11eb4cff.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_002146d58462dcf1138f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_002146d58462dcf1138f.png new file mode 100644 index 0000000000000000000000000000000000000000..4c3fcccd80c1e2ae01603172fbe9ea03fdb10f62 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_002146d58462dcf1138f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_4b64689fc286b148b39a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_4b64689fc286b148b39a.png new file mode 100644 index 0000000000000000000000000000000000000000..cb7f3b55252de71127956e81a55a1426d2d000c6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_4b64689fc286b148b39a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_642de8ca4837518430ae.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_642de8ca4837518430ae.png new file mode 100644 index 0000000000000000000000000000000000000000..482b0d9694a9121cf3a1355a8a274ffeb83fb867 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_642de8ca4837518430ae.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_7fce802a62c03f349904.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_7fce802a62c03f349904.png new file mode 100644 index 0000000000000000000000000000000000000000..a4e43af1d3383ab4870695db79677672e91f5046 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_80_7fce802a62c03f349904.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_061281f0958189cb5d31.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_061281f0958189cb5d31.png new file mode 100644 index 0000000000000000000000000000000000000000..fbe0b3d20cea07301e7e0a3fbffa0f25c5a45a15 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_061281f0958189cb5d31.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_5b6b74a7d7fd49b3636b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_5b6b74a7d7fd49b3636b.png new file mode 100644 index 0000000000000000000000000000000000000000..9cf19a378f905f68c3b2e0cd4e5daf6acbf5e29c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_5b6b74a7d7fd49b3636b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_69329d4e2a0a472e0d04.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_69329d4e2a0a472e0d04.png new file mode 100644 index 0000000000000000000000000000000000000000..ebc9ec5c9ca29841201b0ed4a3d2774c00f29e1f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_69329d4e2a0a472e0d04.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_e9e3f32b220e23c697cf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_e9e3f32b220e23c697cf.png new file mode 100644 index 0000000000000000000000000000000000000000..535b61fdbfba0582ebf0849e7a820a8ed472dfed Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_816_e9e3f32b220e23c697cf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_27f999fd2d618c705cb1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_27f999fd2d618c705cb1.png new file mode 100644 index 0000000000000000000000000000000000000000..8923c7af47dfdadbaa261b62a441d67cfa1bb9d0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_27f999fd2d618c705cb1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_40304dd8f904f4d4b176.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_40304dd8f904f4d4b176.png new file mode 100644 index 0000000000000000000000000000000000000000..e80a480f90722ff8629c0d8de47b171f21020a3f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_40304dd8f904f4d4b176.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_9c2a756ba8f9a949c00e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_9c2a756ba8f9a949c00e.png new file mode 100644 index 0000000000000000000000000000000000000000..3d41c00668e814dad00518ca9640e12fc88ee276 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_9c2a756ba8f9a949c00e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_e40da72508d9965ce259.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_e40da72508d9965ce259.png new file mode 100644 index 0000000000000000000000000000000000000000..a2673c6e0d2ba2c823e013bdd8ec9e0f86e924f3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_824_e40da72508d9965ce259.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_262acb7fdbd40f4b9740.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_262acb7fdbd40f4b9740.png new file mode 100644 index 0000000000000000000000000000000000000000..f4179288e3dcdd5b1dfc1fb2e096b5f2e6bdee72 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_262acb7fdbd40f4b9740.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_306e72d0667b956b2311.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_306e72d0667b956b2311.png new file mode 100644 index 0000000000000000000000000000000000000000..3dc2196f98115c8f52a9d30c6524e78f89353ab4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_306e72d0667b956b2311.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_430e5d6128d45a25cb6f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_430e5d6128d45a25cb6f.png new file mode 100644 index 0000000000000000000000000000000000000000..532b865d64b983723d3978f78ee03d94412da77c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_430e5d6128d45a25cb6f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_7645690b8d91851e7d9d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_7645690b8d91851e7d9d.png new file mode 100644 index 0000000000000000000000000000000000000000..b191e22dfab5812c2d0729bca642d869bb11f26f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_832_7645690b8d91851e7d9d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_2be7449ad230441219d7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_2be7449ad230441219d7.png new file mode 100644 index 0000000000000000000000000000000000000000..b3363e2ff1eaf55ec2b4c75c64e314b3735fde18 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_2be7449ad230441219d7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_b38629e31548571aa3be.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_b38629e31548571aa3be.png new file mode 100644 index 0000000000000000000000000000000000000000..75ecca66c1134c0694b36767b78cb90c3cf7a0ab Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_b38629e31548571aa3be.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_e7fcf8abc3886e356bb8.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_e7fcf8abc3886e356bb8.png new file mode 100644 index 0000000000000000000000000000000000000000..2b6d2ffbcaf2f47280e34abe025a5c80a2e6b1f4 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_e7fcf8abc3886e356bb8.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_ffd650d33cbd150f9d5a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_ffd650d33cbd150f9d5a.png new file mode 100644 index 0000000000000000000000000000000000000000..5758c4bfddacadea365ee4a063ef9b9819afab60 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_840_ffd650d33cbd150f9d5a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_0da7b9c733de297e7940.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_0da7b9c733de297e7940.png new file mode 100644 index 0000000000000000000000000000000000000000..394eb07b3f0ac5327a12092db57e2c7ec4d8f9c6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_0da7b9c733de297e7940.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_7e99547529310b4b5dbe.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_7e99547529310b4b5dbe.png new file mode 100644 index 0000000000000000000000000000000000000000..8fabde6bc7ba135e2a7c280df85e3af8625f5d44 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_7e99547529310b4b5dbe.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_9fd46f5e0289db110e5a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_9fd46f5e0289db110e5a.png new file mode 100644 index 0000000000000000000000000000000000000000..7a6eeb57e1873af7ffa7adfe38e642e8ec56297b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_9fd46f5e0289db110e5a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_d3851e1b7aee0085f878.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_d3851e1b7aee0085f878.png new file mode 100644 index 0000000000000000000000000000000000000000..c55644d37caf5b88db2f815080aa0b71904a1251 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_848_d3851e1b7aee0085f878.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_34456aced927b6fdcddf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_34456aced927b6fdcddf.png new file mode 100644 index 0000000000000000000000000000000000000000..0f5acd31c3b39687c35e24db4160525b52ab69eb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_34456aced927b6fdcddf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_738cae12b93e3de8b6ad.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_738cae12b93e3de8b6ad.png new file mode 100644 index 0000000000000000000000000000000000000000..7e020698d6a4542c655476895277729bbb933763 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_738cae12b93e3de8b6ad.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_a9c51bde36c7b91e7157.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_a9c51bde36c7b91e7157.png new file mode 100644 index 0000000000000000000000000000000000000000..6ca13c52cb69e4a2d6cceb61c4b132e54358c88e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_a9c51bde36c7b91e7157.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_b93f1ea746e64027d9cf.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_b93f1ea746e64027d9cf.png new file mode 100644 index 0000000000000000000000000000000000000000..ad74c99ced0ec45806b80f5b83496f0330e62bd2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_856_b93f1ea746e64027d9cf.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_5a6f78f0d182d46a8beb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_5a6f78f0d182d46a8beb.png new file mode 100644 index 0000000000000000000000000000000000000000..8f015e2da9f60d20f54edd51f97a7a177ac8716a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_5a6f78f0d182d46a8beb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_8aa45d21612bc1191b24.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_8aa45d21612bc1191b24.png new file mode 100644 index 0000000000000000000000000000000000000000..1de7a978717819fce8dca4fc6d92296709db4e48 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_8aa45d21612bc1191b24.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_8ecbb274e51eb741dfb5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_8ecbb274e51eb741dfb5.png new file mode 100644 index 0000000000000000000000000000000000000000..52a69fdc4ecb72e51fdfb562857719c7aec1be39 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_8ecbb274e51eb741dfb5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_950acf7e39a1d4e12f7d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_950acf7e39a1d4e12f7d.png new file mode 100644 index 0000000000000000000000000000000000000000..738068c6c5aa9c54917fe258c5c2516797b1d748 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_864_950acf7e39a1d4e12f7d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_72610369def04cbca766.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_72610369def04cbca766.png new file mode 100644 index 0000000000000000000000000000000000000000..0f8295db5b09b08fa9f24047ff77772679119321 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_72610369def04cbca766.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_92a3a20861d1d9a07c22.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_92a3a20861d1d9a07c22.png new file mode 100644 index 0000000000000000000000000000000000000000..3bb8f5ca5af3fcd11d783b2c477587fdcdd7513b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_92a3a20861d1d9a07c22.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_a69cb09077cee2f55c62.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_a69cb09077cee2f55c62.png new file mode 100644 index 0000000000000000000000000000000000000000..e96a0f65845547394cd12af689a17fe7b134ebf8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_a69cb09077cee2f55c62.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_ca57733755532b74591b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_ca57733755532b74591b.png new file mode 100644 index 0000000000000000000000000000000000000000..5b75412894ec5168b63d00449a23ab6f4f8a6648 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_872_ca57733755532b74591b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_18fe769afdeec594e0f0.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_18fe769afdeec594e0f0.png new file mode 100644 index 0000000000000000000000000000000000000000..22cabf0b5d724290844f72bd2f6ec31afaa7727e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_18fe769afdeec594e0f0.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_2ab57feb81125c56c260.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_2ab57feb81125c56c260.png new file mode 100644 index 0000000000000000000000000000000000000000..7854f02d9ab9fb6fb69b25556882d8996f4d5312 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_2ab57feb81125c56c260.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_3bf88939232b65d3898d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_3bf88939232b65d3898d.png new file mode 100644 index 0000000000000000000000000000000000000000..0d37a1fe5a12cb4a5ca728ecba5e8c6062ebf469 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_3bf88939232b65d3898d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_4ca27ee28456bc10a80b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_4ca27ee28456bc10a80b.png new file mode 100644 index 0000000000000000000000000000000000000000..4ce3edfc997eecf8afed8ad1abfa1f4d7498678e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_880_4ca27ee28456bc10a80b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_19b900daab11785f4793.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_19b900daab11785f4793.png new file mode 100644 index 0000000000000000000000000000000000000000..a9ddc0bf0a4e2eb4b9ca1536455ddaeaab96e480 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_19b900daab11785f4793.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_8729ef5a4b0fe41acc66.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_8729ef5a4b0fe41acc66.png new file mode 100644 index 0000000000000000000000000000000000000000..29a9b300efc29f5044db36b2344af57ea0459366 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_8729ef5a4b0fe41acc66.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_a7440f71b9ffcf04c59e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_a7440f71b9ffcf04c59e.png new file mode 100644 index 0000000000000000000000000000000000000000..8a2133bcfb81636b0595a2f7a5dc2162d2c2bfc8 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_a7440f71b9ffcf04c59e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_aec0fa4930afae7a7541.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_aec0fa4930afae7a7541.png new file mode 100644 index 0000000000000000000000000000000000000000..91414ce073d1ec5b7e55d2f595fb09d4dce6e90f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_888_aec0fa4930afae7a7541.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_88384511781630fcde4e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_88384511781630fcde4e.png new file mode 100644 index 0000000000000000000000000000000000000000..d63cae98244c460c54f59aca428bc532c0d2ce69 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_88384511781630fcde4e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_a4422b8ecb356c655d3a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_a4422b8ecb356c655d3a.png new file mode 100644 index 0000000000000000000000000000000000000000..bc27c2ec6c3b00a84b8c49c0ddbbe47ee097336b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_a4422b8ecb356c655d3a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_b86a7ba09e3469d620dc.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_b86a7ba09e3469d620dc.png new file mode 100644 index 0000000000000000000000000000000000000000..a2ac45f0762ac5d839d2ecee2454dc80a8edf64e Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_b86a7ba09e3469d620dc.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_d1da96f2064c4210048a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_d1da96f2064c4210048a.png new file mode 100644 index 0000000000000000000000000000000000000000..add25c036f620ee162610a4ff930aec7b5bb566f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_88_d1da96f2064c4210048a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_1195075f26c9c85c17be.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_1195075f26c9c85c17be.png new file mode 100644 index 0000000000000000000000000000000000000000..60a384bf384447d6e35251279b807150cf7ccb94 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_1195075f26c9c85c17be.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_2d49f7aa4f79dc70f56e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_2d49f7aa4f79dc70f56e.png new file mode 100644 index 0000000000000000000000000000000000000000..72d024d8d267d09f0229e34073e4bad8b73e0b3f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_2d49f7aa4f79dc70f56e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_8b01143d063fb3d088dd.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_8b01143d063fb3d088dd.png new file mode 100644 index 0000000000000000000000000000000000000000..df31039e28ea10525cb998cced70a2589bd7be20 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_8b01143d063fb3d088dd.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_a33d92e49bb3819c9398.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_a33d92e49bb3819c9398.png new file mode 100644 index 0000000000000000000000000000000000000000..b2ed22f2cb816b7083fd28c4e88890aa83064b8d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_896_a33d92e49bb3819c9398.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_018b5a46c57a4e62ee91.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_018b5a46c57a4e62ee91.png new file mode 100644 index 0000000000000000000000000000000000000000..a0ddef301f2992825c9fdfdc1339f4b59893d962 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_018b5a46c57a4e62ee91.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_43f961633a45946caf61.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_43f961633a45946caf61.png new file mode 100644 index 0000000000000000000000000000000000000000..df3c3b7be3a173011f1cae19cc87d26c6971238b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_43f961633a45946caf61.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_4f431957962c49867234.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_4f431957962c49867234.png new file mode 100644 index 0000000000000000000000000000000000000000..a7e3c38526bf0ad984e4244a972146a8d3f9339f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_4f431957962c49867234.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_b1af5834223b938d847b.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_b1af5834223b938d847b.png new file mode 100644 index 0000000000000000000000000000000000000000..d96da9af4674b25ff04eb05bf130ac89e65306e2 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_8_b1af5834223b938d847b.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_01099aeefecc508a7281.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_01099aeefecc508a7281.png new file mode 100644 index 0000000000000000000000000000000000000000..0a0e24ab01cefa6de8f4687bc3bc91ac7b2aae1c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_01099aeefecc508a7281.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_74a58ccedcd0aba02d9d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_74a58ccedcd0aba02d9d.png new file mode 100644 index 0000000000000000000000000000000000000000..6a9d0b34fb5abbefa79b6021829a51876fd87115 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_74a58ccedcd0aba02d9d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_d08a528486fd9d24317e.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_d08a528486fd9d24317e.png new file mode 100644 index 0000000000000000000000000000000000000000..7fbe4419b123184bb9f69e64a19a76643164c109 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_d08a528486fd9d24317e.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_d9f4aaa16030ef084b93.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_d9f4aaa16030ef084b93.png new file mode 100644 index 0000000000000000000000000000000000000000..51b8ce5249b989b0c6d9bb5bb17e9a0699d4093d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_904_d9f4aaa16030ef084b93.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_4a51dc0e598ef75ade74.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_4a51dc0e598ef75ade74.png new file mode 100644 index 0000000000000000000000000000000000000000..7e2cf4c2d80ce29f090fd2712823789a01913220 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_4a51dc0e598ef75ade74.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_7c4eaf576d54dc0892c4.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_7c4eaf576d54dc0892c4.png new file mode 100644 index 0000000000000000000000000000000000000000..a9aea3955ebecd0d3cfe745111253b1ecde47321 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_7c4eaf576d54dc0892c4.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_87ef29d0d2134d35550f.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_87ef29d0d2134d35550f.png new file mode 100644 index 0000000000000000000000000000000000000000..456b43a80b9c85b7497d5ffda63c81d88d35b437 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_87ef29d0d2134d35550f.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_ac2d97b761361e2c7f95.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_ac2d97b761361e2c7f95.png new file mode 100644 index 0000000000000000000000000000000000000000..28ac085ddd9cc675c47557d97c3c7346ae577db3 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_912_ac2d97b761361e2c7f95.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_5b7b240cdae3212a828d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_5b7b240cdae3212a828d.png new file mode 100644 index 0000000000000000000000000000000000000000..1f8fd7ba55723954dbb2391ef6dc59d0a7d214bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_5b7b240cdae3212a828d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_cb802cc8bb9421190135.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_cb802cc8bb9421190135.png new file mode 100644 index 0000000000000000000000000000000000000000..608965ca78832854b322b5a67ec380a29052d82b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_cb802cc8bb9421190135.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_ceb5dd8574ce0588ad3d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_ceb5dd8574ce0588ad3d.png new file mode 100644 index 0000000000000000000000000000000000000000..f8601022da26b12b25eb8be71327cc0c7194f9f0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_ceb5dd8574ce0588ad3d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_dd362beabef9f0f07f1c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_dd362beabef9f0f07f1c.png new file mode 100644 index 0000000000000000000000000000000000000000..a29ee5b938536f94c29e84775c6540da6c212e95 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_920_dd362beabef9f0f07f1c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_0ccf3153354added95d1.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_0ccf3153354added95d1.png new file mode 100644 index 0000000000000000000000000000000000000000..d106269fa0af773c5f692c9888e861a76d190aec Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_0ccf3153354added95d1.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_35f2e6d732e429b27cef.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_35f2e6d732e429b27cef.png new file mode 100644 index 0000000000000000000000000000000000000000..421d3fd2b73cce96c9920980de1948d7d58a8778 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_35f2e6d732e429b27cef.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_6c25bd7611e598bbfbe5.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_6c25bd7611e598bbfbe5.png new file mode 100644 index 0000000000000000000000000000000000000000..fd32521299b82c2f13c08cbeb8a7176bbeac9246 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_6c25bd7611e598bbfbe5.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_c7d072dca160d6de598d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_c7d072dca160d6de598d.png new file mode 100644 index 0000000000000000000000000000000000000000..d90c21b4714877a17675acae2f254f3f01405d44 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_928_c7d072dca160d6de598d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_14f68a769ca53b61b6be.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_14f68a769ca53b61b6be.png new file mode 100644 index 0000000000000000000000000000000000000000..280b00e0c3b9adaf5b822b2ea989f6726a432143 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_14f68a769ca53b61b6be.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_190b7cee3f4ce931d581.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_190b7cee3f4ce931d581.png new file mode 100644 index 0000000000000000000000000000000000000000..e4ac988f727733632fc6727912ea7d0b9d6813b6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_190b7cee3f4ce931d581.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_427cd756a6fbf28d7994.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_427cd756a6fbf28d7994.png new file mode 100644 index 0000000000000000000000000000000000000000..f6c707006b6b6249b01fffaf9186a0373f2564bb Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_427cd756a6fbf28d7994.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_fbfe70b4fd3580e4922a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_fbfe70b4fd3580e4922a.png new file mode 100644 index 0000000000000000000000000000000000000000..70b0ce6d482908fc4e50e11d887f8173686bb2de Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_936_fbfe70b4fd3580e4922a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_131e7280660d61568c15.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_131e7280660d61568c15.png new file mode 100644 index 0000000000000000000000000000000000000000..0d0c05eb8e8ab3b3c010d749c52a7686f1f672b6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_131e7280660d61568c15.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_41a5dce9cfa5cf744e71.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_41a5dce9cfa5cf744e71.png new file mode 100644 index 0000000000000000000000000000000000000000..cc0c08bc648ed471d460e92bc8f323cd78034701 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_41a5dce9cfa5cf744e71.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_8d33167a8809bc5e2e10.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_8d33167a8809bc5e2e10.png new file mode 100644 index 0000000000000000000000000000000000000000..248ba4107a383cae023ead284d1a1f5abd826644 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_8d33167a8809bc5e2e10.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_d7a08791c37a0d7e716d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_d7a08791c37a0d7e716d.png new file mode 100644 index 0000000000000000000000000000000000000000..9abb717a2f675e00131530182ba6e4e8ff6d1f12 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_944_d7a08791c37a0d7e716d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_49c73fa7e966791f5390.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_49c73fa7e966791f5390.png new file mode 100644 index 0000000000000000000000000000000000000000..2cc1b088075244405e806fc89246090c91daa54c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_49c73fa7e966791f5390.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_c88dcfd62ec0e16b152a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_c88dcfd62ec0e16b152a.png new file mode 100644 index 0000000000000000000000000000000000000000..8b7e5323f8927395e73931548db4785c5b8c7ff7 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_c88dcfd62ec0e16b152a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_e9c245f08754c65b88da.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_e9c245f08754c65b88da.png new file mode 100644 index 0000000000000000000000000000000000000000..4496ab7903fb1a5ba4654c7a8e6b8ef0dfce168d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_e9c245f08754c65b88da.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_fc214fd3c543e1062168.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_fc214fd3c543e1062168.png new file mode 100644 index 0000000000000000000000000000000000000000..a30586d43bb7cbec84c514cf5ab9606e411dfebc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_952_fc214fd3c543e1062168.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_1f1bb4f5e11489979814.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_1f1bb4f5e11489979814.png new file mode 100644 index 0000000000000000000000000000000000000000..e2259fe6db55fc2cc9772e5dd180a7909d650b4d Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_1f1bb4f5e11489979814.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_bbcff34d61dc915cfffa.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_bbcff34d61dc915cfffa.png new file mode 100644 index 0000000000000000000000000000000000000000..9424c979f6d08ca554ae29244c385e987c76c708 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_bbcff34d61dc915cfffa.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_d943310ca37254daf167.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_d943310ca37254daf167.png new file mode 100644 index 0000000000000000000000000000000000000000..dece639015596d199adb6e9180c58fdfe1df8e4f Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_d943310ca37254daf167.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_dab10b88bdf06be619c2.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_dab10b88bdf06be619c2.png new file mode 100644 index 0000000000000000000000000000000000000000..b6208b2b427ba0cd131277d25a76f598d4fe66e5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_960_dab10b88bdf06be619c2.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_37b53ef8f7963631c78a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_37b53ef8f7963631c78a.png new file mode 100644 index 0000000000000000000000000000000000000000..7326333e3c5bb5614175dafd7eebbde7e17eec56 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_37b53ef8f7963631c78a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_73e3db3d1e8039707146.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_73e3db3d1e8039707146.png new file mode 100644 index 0000000000000000000000000000000000000000..c48539741e7bfdca5f1f5fe75bf7fbb15bb26011 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_73e3db3d1e8039707146.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_d57c92e936cc34db3b66.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_d57c92e936cc34db3b66.png new file mode 100644 index 0000000000000000000000000000000000000000..f3f9d61d4821971800ebcf8a1fb61594018b4942 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_d57c92e936cc34db3b66.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_d874c01a95067c21bdbb.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_d874c01a95067c21bdbb.png new file mode 100644 index 0000000000000000000000000000000000000000..848a326417060306b343a36cecccc92901558822 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_968_d874c01a95067c21bdbb.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_0d776d95eff027dba77c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_0d776d95eff027dba77c.png new file mode 100644 index 0000000000000000000000000000000000000000..0f51fcdf37f61ab817474aeb88d4b4869528b611 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_0d776d95eff027dba77c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_298b674dbb75f0e2d399.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_298b674dbb75f0e2d399.png new file mode 100644 index 0000000000000000000000000000000000000000..0552a6990d517a1ae5441ba2dda0237bed695ba9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_298b674dbb75f0e2d399.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_632ab0608582a5d13e60.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_632ab0608582a5d13e60.png new file mode 100644 index 0000000000000000000000000000000000000000..96bb1eb48a0794b4d061b1cb20e79c553e2be7e9 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_632ab0608582a5d13e60.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_f4754be0e0fb99c57d60.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_f4754be0e0fb99c57d60.png new file mode 100644 index 0000000000000000000000000000000000000000..dd7fd4226e686e72099f2de914ed07ecc4bdfd06 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_96_f4754be0e0fb99c57d60.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_1c6ecb09cf45729ec02d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_1c6ecb09cf45729ec02d.png new file mode 100644 index 0000000000000000000000000000000000000000..866df77a089936ee9e18eb42879edf70b0fc2b77 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_1c6ecb09cf45729ec02d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_24834f9350949b5f8e1c.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_24834f9350949b5f8e1c.png new file mode 100644 index 0000000000000000000000000000000000000000..78238d1affc3bde5cef4240083c06147b5610d3a Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_24834f9350949b5f8e1c.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_96467eb797af222d5127.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_96467eb797af222d5127.png new file mode 100644 index 0000000000000000000000000000000000000000..fd127f29bdf00f24459c6982f3459dc371e062b0 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_96467eb797af222d5127.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_b1d4d79c192e807ea9ea.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_b1d4d79c192e807ea9ea.png new file mode 100644 index 0000000000000000000000000000000000000000..ad91f0876f964420e3f5cb1059c6e9245a87eb9b Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_976_b1d4d79c192e807ea9ea.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_6babdd3a11961e590892.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_6babdd3a11961e590892.png new file mode 100644 index 0000000000000000000000000000000000000000..eac8b1b4c11650459580f5907e16a1cde0607d77 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_6babdd3a11961e590892.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_858c44737753f571d83a.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_858c44737753f571d83a.png new file mode 100644 index 0000000000000000000000000000000000000000..8a0537b5ccf17b7a099e9e9760d0e863f4e2fdcd Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_858c44737753f571d83a.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_8655855a41d6a76b3f50.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_8655855a41d6a76b3f50.png new file mode 100644 index 0000000000000000000000000000000000000000..fa423f7ab7b0514c031953a8d1cc170bbe365f7c Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_8655855a41d6a76b3f50.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_8ce2b6b9e57abff6242d.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_8ce2b6b9e57abff6242d.png new file mode 100644 index 0000000000000000000000000000000000000000..7bddc7249c447fc1bec84ec23861480d12693d18 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_984_8ce2b6b9e57abff6242d.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_297cfc9d19c6bf373211.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_297cfc9d19c6bf373211.png new file mode 100644 index 0000000000000000000000000000000000000000..f0f7ef2e12b9ddc0b41cdd9d72d0c7534f8367bc Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_297cfc9d19c6bf373211.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_81d31deea5f321a879ea.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_81d31deea5f321a879ea.png new file mode 100644 index 0000000000000000000000000000000000000000..8b49627e7afaf5abbe8abe310be75e23dc4ed6c5 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_81d31deea5f321a879ea.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_b832ae939713ff831840.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_b832ae939713ff831840.png new file mode 100644 index 0000000000000000000000000000000000000000..e625e53799ba8610f48b0a510a7b9e8252d89310 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_b832ae939713ff831840.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_d3be1fc1429f13f91ba7.png b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_d3be1fc1429f13f91ba7.png new file mode 100644 index 0000000000000000000000000000000000000000..1ccbc573fd61d2db46b90a8fb1673b8a72e0abd6 Binary files /dev/null and b/wandb/run-20230711_182238-k29fqnbt/files/media/images/validation_992_d3be1fc1429f13f91ba7.png differ diff --git a/wandb/run-20230711_182238-k29fqnbt/files/output.log b/wandb/run-20230711_182238-k29fqnbt/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c30753185b24327f607c6dcff1f42a89ef441927 --- /dev/null +++ b/wandb/run-20230711_182238-k29fqnbt/files/output.log @@ -0,0 +1,4075 @@ +07/11/2023 18:22:40 - INFO - __main__ - Distributed environment: MULTI_GPU Backend: nccl +Num processes: 4 +Process index: 0 +Local process index: 0 +Device: cuda:0 +Mixed precision type: fp16 +{'thresholding', 'sample_max_value', 'dynamic_thresholding_ratio', 'variance_type', 'prediction_type', 'timestep_spacing', 'clip_sample_range'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +{'num_class_embeds', 'timestep_post_act', 'class_embed_type', 'conv_out_kernel', 'only_cross_attention', 'time_cond_proj_dim', 'dual_cross_attention', 'addition_embed_type', 'projection_class_embeddings_input_dim', 'class_embeddings_concat', 'mid_block_only_cross_attention', 'time_embedding_act_fn', 'num_attention_heads', 'encoder_hid_dim', 'cross_attention_norm', 'mid_block_type', 'resnet_time_scale_shift', 'addition_embed_type_num_heads', 'resnet_out_scale_factor', 'time_embedding_dim', 'time_embedding_type', 'conv_in_kernel', 'resnet_skip_time_act', 'encoder_hid_dim_type', 'upcast_attention', 'use_linear_projection'} was not found in config. Values will be initialized to default values. +Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 45428.54it/s] +07/11/2023 18:22:45 - WARNING - datasets.builder - Found cached dataset imagefolder (/l/vision/v5/sragas/hf_models/imagefolder/default-651344a1f880f9c2/0.0.0/37fbb85cc714a338bea574ac6c7d0b5be5aff46c1862c1989b20e0771199e93f) +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 373.16it/s] +07/11/2023 18:22:48 - INFO - __main__ - ***** Running training ***** +07/11/2023 18:22:48 - INFO - __main__ - Num examples = 228 +07/11/2023 18:22:48 - INFO - __main__ - Num Epochs = 300 +07/11/2023 18:22:48 - INFO - __main__ - Instantaneous batch size per device = 2 +07/11/2023 18:22:48 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 32 +07/11/2023 18:22:48 - INFO - __main__ - Gradient Accumulation steps = 4 +07/11/2023 18:22:48 - INFO - __main__ - Total optimization steps = 2400 + + + + + + + + + + + + +Steps: 0%|▎ | 8/2400 [00:25<1:40:49, 2.53s/it, lr=1.6e-7, step_loss=0.141]07/11/2023 18:23:13 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 1%|▌ | 16/2400 [01:21<2:07:46, 3.22s/it, lr=3.2e-7, step_loss=0.128]07/11/2023 18:24:09 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 1%|▉ | 24/2400 [01:59<1:50:24, 2.79s/it, lr=4.8e-7, step_loss=0.00693]07/11/2023 18:24:47 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 1%|█▏ | 32/2400 [02:36<1:42:47, 2.60s/it, lr=6.4e-7, step_loss=0.0381]07/11/2023 18:25:24 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 2%|█▌ | 40/2400 [03:13<1:52:03, 2.85s/it, lr=8e-7, step_loss=0.0652]07/11/2023 18:26:01 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 2%|█▊ | 48/2400 [03:49<1:43:49, 2.65s/it, lr=9.6e-7, step_loss=0.123]07/11/2023 18:26:37 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 2%|██ | 56/2400 [04:26<1:46:46, 2.73s/it, lr=1.12e-6, step_loss=0.121]07/11/2023 18:27:14 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 3%|██▍ | 64/2400 [05:03<1:43:29, 2.66s/it, lr=1.28e-6, step_loss=0.0359]07/11/2023 18:27:51 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 3%|██▋ | 72/2400 [05:39<1:44:57, 2.71s/it, lr=1.44e-6, step_loss=0.0288]07/11/2023 18:28:27 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 3%|███ | 80/2400 [06:16<1:46:02, 2.74s/it, lr=1.6e-6, step_loss=0.0706]07/11/2023 18:29:04 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 4%|███▎ | 88/2400 [06:53<1:42:05, 2.65s/it, lr=1.76e-6, step_loss=0.39]07/11/2023 18:29:41 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 4%|███▋ | 96/2400 [07:32<1:47:27, 2.80s/it, lr=1.92e-6, step_loss=0.189]07/11/2023 18:30:20 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 4%|███▊ | 104/2400 [08:08<1:44:55, 2.74s/it, lr=2.08e-6, step_loss=0.0172]07/11/2023 18:30:56 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 5%|████▏ | 112/2400 [08:44<1:42:18, 2.68s/it, lr=2.24e-6, step_loss=0.284]07/11/2023 18:31:32 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + +Steps: 5%|████▌ | 120/2400 [09:20<1:38:28, 2.59s/it, lr=2.4e-6, step_loss=0.0761]07/11/2023 18:32:08 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 5%|████▊ | 128/2400 [09:56<1:40:08, 2.64s/it, lr=2.56e-6, step_loss=0.33]07/11/2023 18:32:44 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 6%|█████ | 136/2400 [10:34<1:46:03, 2.81s/it, lr=2.72e-6, step_loss=0.293]07/11/2023 18:33:22 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 6%|█████▍ | 144/2400 [11:10<1:39:19, 2.64s/it, lr=2.88e-6, step_loss=0.086]07/11/2023 18:33:58 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 6%|█████▊ | 152/2400 [11:47<1:42:04, 2.72s/it, lr=3.04e-6, step_loss=0.2]07/11/2023 18:34:35 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 7%|██████ | 160/2400 [12:23<1:36:04, 2.57s/it, lr=3.2e-6, step_loss=0.366]07/11/2023 18:35:11 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 7%|██████▏ | 168/2400 [13:00<1:38:31, 2.65s/it, lr=3.36e-6, step_loss=0.0729]07/11/2023 18:35:48 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 7%|██████▋ | 176/2400 [13:38<1:52:41, 3.04s/it, lr=3.52e-6, step_loss=0.15]07/11/2023 18:36:26 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 8%|██████▊ | 184/2400 [14:15<1:35:32, 2.59s/it, lr=3.68e-6, step_loss=0.0309]07/11/2023 18:37:03 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 8%|███████ | 192/2400 [14:51<1:39:45, 2.71s/it, lr=3.84e-6, step_loss=0.0319]07/11/2023 18:37:39 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 8%|███████▊ | 200/2400 [15:29<1:42:42, 2.80s/it, lr=4e-6, step_loss=0.031]07/11/2023 18:38:17 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 9%|███████▋ | 208/2400 [16:05<1:38:55, 2.71s/it, lr=4.16e-6, step_loss=0.0664]07/11/2023 18:38:53 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 9%|████████ | 216/2400 [16:43<1:48:03, 2.97s/it, lr=4.32e-6, step_loss=0.0207]07/11/2023 18:39:31 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 9%|████████▎ | 224/2400 [17:21<1:45:05, 2.90s/it, lr=4.48e-6, step_loss=0.0788]07/11/2023 18:40:09 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 10%|████████▋ | 232/2400 [17:58<1:38:14, 2.72s/it, lr=4.64e-6, step_loss=0.258]07/11/2023 18:40:46 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 10%|█████████ | 240/2400 [18:34<1:40:18, 2.79s/it, lr=4.8e-6, step_loss=0.119]07/11/2023 18:41:22 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 10%|█████████▎ | 248/2400 [19:11<1:37:19, 2.71s/it, lr=4.96e-6, step_loss=0.164]07/11/2023 18:41:59 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 11%|█████████▌ | 256/2400 [19:49<1:36:28, 2.70s/it, lr=5.12e-6, step_loss=0.159]07/11/2023 18:42:37 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 11%|█████████▉ | 264/2400 [20:27<1:39:56, 2.81s/it, lr=5.28e-6, step_loss=0.219]07/11/2023 18:43:15 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 11%|██████████▏ | 272/2400 [21:06<1:37:33, 2.75s/it, lr=5.44e-6, step_loss=0.346]07/11/2023 18:43:54 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 12%|██████████▌ | 280/2400 [21:42<1:32:59, 2.63s/it, lr=5.6e-6, step_loss=0.0897]07/11/2023 18:44:30 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 12%|██████████▊ | 288/2400 [22:18<1:28:34, 2.52s/it, lr=5.76e-6, step_loss=0.162]07/11/2023 18:45:06 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 12%|███████████ | 296/2400 [22:56<1:35:53, 2.73s/it, lr=5.92e-6, step_loss=0.184]07/11/2023 18:45:44 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 13%|███████████▏ | 304/2400 [23:33<1:30:01, 2.58s/it, lr=6.08e-6, step_loss=0.00378]07/11/2023 18:46:21 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 13%|███████████▌ | 312/2400 [24:10<1:38:43, 2.84s/it, lr=6.24e-6, step_loss=0.0894]07/11/2023 18:46:58 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 13%|███████████▊ | 320/2400 [24:45<1:26:56, 2.51s/it, lr=6.4e-6, step_loss=0.00816]07/11/2023 18:47:34 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 14%|████████████▎ | 328/2400 [25:23<1:41:22, 2.94s/it, lr=6.56e-6, step_loss=0.074]07/11/2023 18:48:11 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 14%|████████████▌ | 336/2400 [26:00<1:33:34, 2.72s/it, lr=6.72e-6, step_loss=0.651]07/11/2023 18:48:48 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 14%|████████████▉ | 344/2400 [26:37<1:37:51, 2.86s/it, lr=6.88e-6, step_loss=0.101]07/11/2023 18:49:25 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 15%|█████████████▏ | 352/2400 [27:14<1:29:49, 2.63s/it, lr=7.04e-6, step_loss=0.284]07/11/2023 18:50:02 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 15%|█████████████▋ | 360/2400 [27:50<1:31:38, 2.70s/it, lr=7.2e-6, step_loss=0.118]07/11/2023 18:50:38 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 15%|█████████████▋ | 368/2400 [28:28<1:34:56, 2.80s/it, lr=7.36e-6, step_loss=0.0962]07/11/2023 18:51:16 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 16%|██████████████ | 376/2400 [29:07<1:38:11, 2.91s/it, lr=7.52e-6, step_loss=0.241]07/11/2023 18:51:55 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 16%|██████████████▏ | 384/2400 [29:43<1:28:46, 2.64s/it, lr=7.68e-6, step_loss=0.0114]07/11/2023 18:52:31 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 16%|██████████████▌ | 392/2400 [30:21<1:34:21, 2.82s/it, lr=7.84e-6, step_loss=0.0276]07/11/2023 18:53:09 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 17%|███████████████▌ | 400/2400 [30:59<1:34:42, 2.84s/it, lr=8e-6, step_loss=0.328]07/11/2023 18:53:47 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 17%|███████████████▏ | 408/2400 [31:37<1:38:06, 2.95s/it, lr=8.16e-6, step_loss=0.0744]07/11/2023 18:54:25 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + +Steps: 17%|███████████████▍ | 416/2400 [32:14<1:26:47, 2.62s/it, lr=8.32e-6, step_loss=0.0553]07/11/2023 18:55:02 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + + +Steps: 18%|███████████████▉ | 424/2400 [32:52<1:29:26, 2.72s/it, lr=8.48e-6, step_loss=0.012]07/11/2023 18:55:40 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 18%|████████████████ | 432/2400 [33:28<1:26:37, 2.64s/it, lr=8.64e-6, step_loss=0.0132]07/11/2023 18:56:16 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 18%|████████████████▊ | 440/2400 [34:06<1:31:57, 2.82s/it, lr=8.8e-6, step_loss=0.04]07/11/2023 18:56:54 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 19%|████████████████▊ | 448/2400 [34:43<1:35:54, 2.95s/it, lr=8.96e-6, step_loss=0.115]07/11/2023 18:57:31 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 19%|████████████████▉ | 456/2400 [35:21<1:30:14, 2.79s/it, lr=9.12e-6, step_loss=0.0943]07/11/2023 18:58:09 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 19%|█████████████████▍ | 464/2400 [35:58<1:26:48, 2.69s/it, lr=9.28e-6, step_loss=0.299]07/11/2023 18:58:46 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 20%|█████████████████▌ | 472/2400 [36:34<1:27:44, 2.73s/it, lr=9.44e-6, step_loss=0.0703]07/11/2023 18:59:22 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 20%|██████████████████ | 480/2400 [37:11<1:23:54, 2.62s/it, lr=9.6e-6, step_loss=0.0568]07/11/2023 18:59:59 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + +Steps: 20%|██████████████████ | 488/2400 [37:48<1:23:34, 2.62s/it, lr=9.76e-6, step_loss=0.0508]07/11/2023 19:00:36 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + + + + + + + +Steps: 21%|██████████████████▍ | 496/2400 [38:26<1:32:37, 2.92s/it, lr=9.92e-6, step_loss=0.0544]07/11/2023 19:01:14 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. +{'requires_safety_checker'} was not found in config. Values will be initialized to default values. +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values. +{'scaling_factor'} was not found in config. Values will be initialized to default values. +`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden. + + + + + +Steps: 21%|██████████████████▌ | 500/2400 [38:52<2:23:37, 4.54s/it, lr=9.98e-6, step_loss=0.0462]07/11/2023 19:01:40 - INFO - accelerate.accelerator - Saving current state to /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500 +07/11/2023 19:01:40 - INFO - accelerate.checkpointing - Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500/pytorch_model.bin +07/11/2023 19:01:40 - INFO - accelerate.checkpointing - Optimizer state saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500/optimizer.bin +07/11/2023 19:01:40 - INFO - accelerate.checkpointing - Scheduler state saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500/scheduler.bin +07/11/2023 19:01:40 - INFO - accelerate.checkpointing - Gradient scaler state saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500/scaler.pt +07/11/2023 19:01:40 - INFO - accelerate.checkpointing - Random states saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500/random_states_0.pkl +07/11/2023 19:01:40 - INFO - __main__ - Saved state to /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-500 +Steps: 21%|███████████████████▏ | 500/2400 [38:54<2:23:37, 4.54s/it, lr=1e-5, step_loss=0.0898] +Steps: 21%|███████████████████▍ | 501/2400 [38:56<2:14:17, 4.24s/it, lr=1e-5, step_loss=0.156] +Steps: 21%|███████████████████▍ | 502/2400 [38:58<2:02:03, 3.86s/it, lr=1e-5, step_loss=0.101] +Steps: 21%|███████████████████▍ | 502/2400 [39:00<2:02:03, 3.86s/it, lr=1e-5, step_loss=0.115] +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +Steps: 21%|███████████████████▌ | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. | 504/2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values./2400 [39:02<1:25:07, 2.69s/it, lr=1e-5, step_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +07/11/2023 20:22:07 - INFO - accelerate.checkpointing - Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +07/11/2023 20:22:07 - INFO - accelerate.checkpointing - Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +07/11/2023 20:22:07 - INFO - accelerate.checkpointing - Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +07/11/2023 20:22:07 - INFO - accelerate.checkpointing - Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +07/11/2023 20:22:07 - INFO - accelerate.checkpointing - Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-1500/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up. Model weights saved in /l/vision/v5/sragas/easel_ai/models_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +{'timestep_spacing', 'prediction_type'} was not found in config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... + Generating 4 images with prompt: tom cruise thumbs up.n config. Values will be initialized to default values.dels_plain/checkpoint-2000/pytorch_model.binep_loss=0.275]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +scheduler.bin: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 563/563 [00:00<00:00, 2.64kB/s]07/11/2023 19:01:50 - INFO - __main__ - Running validation... +run-wt8le4qg.wandb: 0%| | 0.00/1.25M [00:00 + main() + File "train_text_to_image_lora.py", line 906, in main + upload_folder( + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/hf_api.py", line 826, in _inner + return fn(self, *args, **kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/hf_api.py", line 3493, in upload_folder + commit_info = self.create_commit( + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/hf_api.py", line 826, in _inner + return fn(self, *args, **kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/hf_api.py", line 2680, in create_commit + upload_lfs_files( + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/_commit_api.py", line 353, in upload_lfs_files + thread_map( + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/tqdm/contrib/concurrent.py", line 69, in thread_map + return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map + return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs)) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/tqdm/std.py", line 1178, in __iter__ + for obj in iterable: + File "/usr/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator + yield fs.pop().result() + File "/usr/lib/python3.8/concurrent/futures/_base.py", line 437, in result + return self.__get_result() + File "/usr/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result + raise self._exception + File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57, in run + result = self.fn(*self.args, **self.kwargs) + File "/nfs/blitzle/home/data/vision5/sragas/easel_venv/lib/python3.8/site-packages/huggingface_hub/_commit_api.py", line 343, in _wrapped_lfs_upload + raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc +RuntimeError: Error while uploading 'wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb' to the Hub. \ No newline at end of file diff --git a/wandb/run-20230712_000308-yuype6tc/files/requirements.txt b/wandb/run-20230712_000308-yuype6tc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..631710e9e888cc8a873b5d74d41732719a57dd54 --- /dev/null +++ b/wandb/run-20230712_000308-yuype6tc/files/requirements.txt @@ -0,0 +1,165 @@ +absl-py==1.4.0 +accelerate==0.20.3 +aiohttp==3.8.4 +aiosignal==1.3.1 +anyio==3.7.0 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==21.3.0 +asttokens==2.2.1 +async-lru==2.0.2 +async-timeout==4.0.2 +attrs==23.1.0 +babel==2.12.1 +backcall==0.2.0 +beautifulsoup4==4.12.2 +bleach==6.0.0 +cachetools==5.3.1 +certifi==2023.5.7 +cffi==1.15.1 +charset-normalizer==3.1.0 +click==8.1.3 +cmake==3.26.4 +comm==0.1.3 +datasets==2.13.1 +debugpy==1.6.7 +decorator==5.1.1 +defusedxml==0.7.1 +diffusers==0.18.0.dev0 +dill==0.3.6 +docker-pycreds==0.4.0 +exceptiongroup==1.1.2 +executing==1.2.0 +fastjsonschema==2.17.1 +filelock==3.12.2 +frozenlist==1.3.3 +fsspec==2023.6.0 +ftfy==6.1.1 +gitdb==4.0.10 +gitpython==3.1.31 +google-auth-oauthlib==1.0.0 +google-auth==2.21.0 +grpcio==1.56.0 +huggingface-hub==0.15.1 +idna==3.4 +importlib-metadata==6.7.0 +importlib-resources==5.12.0 +ipykernel==6.24.0 +ipython==8.12.2 +jedi==0.18.2 +jinja2==3.1.2 +json5==0.9.14 +jsonschema==4.17.3 +jupyter-client==8.3.0 +jupyter-core==5.3.1 +jupyter-events==0.6.3 +jupyter-lsp==2.2.0 +jupyter-server-terminals==0.4.4 +jupyter-server==2.7.0 +jupyterlab-pygments==0.2.2 +jupyterlab-server==2.23.0 +jupyterlab==4.0.2 +lit==16.0.6 +markdown==3.4.3 +markupsafe==2.1.3 +matplotlib-inline==0.1.6 +mistune==3.0.1 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.14 +mypy-extensions==1.0.0 +nbclient==0.8.0 +nbconvert==7.6.0 +nbformat==5.9.0 +nest-asyncio==1.5.6 +networkx==3.1 +notebook-shim==0.2.3 +numpy==1.24.4 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +oauthlib==3.2.2 +overrides==7.3.1 +packaging==23.1 +pandas==2.0.3 +pandocfilters==1.5.0 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==10.0.0 +pip==20.0.2 +pkg-resources==0.0.0 +pkgutil-resolve-name==1.3.10 +platformdirs==3.8.0 +prometheus-client==0.17.0 +prompt-toolkit==3.0.39 +protobuf==4.23.3 +psutil==5.9.5 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==12.0.1 +pyasn1-modules==0.3.0 +pyasn1==0.5.0 +pycparser==2.21 +pygments==2.15.1 +pyre-extensions==0.0.29 +pyrsistent==0.19.3 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +pytz==2023.3 +pyyaml==6.0 +pyzmq==25.1.0 +regex==2023.6.3 +requests-oauthlib==1.3.1 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rsa==4.9 +safetensors==0.3.1 +send2trash==1.8.2 +sentry-sdk==1.27.0 +setproctitle==1.3.2 +setuptools==44.0.0 +six==1.16.0 +smmap==5.0.0 +sniffio==1.3.0 +soupsieve==2.4.1 +stack-data==0.6.2 +sympy==1.12 +tensorboard-data-server==0.7.1 +tensorboard==2.13.0 +terminado==0.17.1 +tinycss2==1.2.1 +tokenizers==0.13.3 +tomli==2.0.1 +torch==2.0.1 +torchaudio==2.0.2 +torchvision==0.15.2 +tornado==6.3.2 +tqdm==4.65.0 +traitlets==5.9.0 +transformers==4.30.2 +triton==2.0.0 +typing-extensions==4.7.1 +typing-inspect==0.9.0 +tzdata==2023.3 +urllib3==2.0.3 +wandb==0.15.4 +wcwidth==0.2.6 +webencodings==0.5.1 +websocket-client==1.6.1 +werkzeug==2.3.6 +wheel==0.40.0 +xformers==0.0.20 +xxhash==3.2.0 +yarl==1.9.2 +zipp==3.15.0 \ No newline at end of file diff --git a/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json b/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4e37bb0dab57f8300cf9d50ba9fce5830a954d57 --- /dev/null +++ b/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json @@ -0,0 +1,127 @@ +{ + "os": "Linux-5.15.0-60-generic-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2023-07-12T04:03:08.519507", + "startedAt": "2023-07-12T04:03:08.061688", + "docker": null, + "cuda": null, + "args": [ + "--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5", + "--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/", + "--resolution=512", + "--center_crop", + "--random_flip", + "--train_batch_size=2", + "--gradient_accumulation_steps=4", + "--num_train_epochs=300", + "--learning_rate=1e-5", + "--max_grad_norm=1", + "--lr_scheduler=cosine", + "--lr_warmup_steps=500", + "--output_dir=/l/vision/v5/sragas/easel_ai/models_plain/", + "--checkpointing_steps=500", + "--report_to=wandb", + "--validation_prompt=tom cruise thumbs up", + "--seed=51", + "--push_to_hub", + "--hub_model_id=person-thumbs-up-plain-lora" + ], + "state": "running", + "program": "train_text_to_image_lora.py", + "codePath": "train_text_to_image_lora.py", + "host": "magneton", + "username": "sragas", + "executable": "/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3", + "cpu_count": 6, + "cpu_count_logical": 12, + "cpu_freq": { + "current": 1779.2625, + "min": 1200.0, + "max": 3700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3100.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1500.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1200.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 2100.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 3100.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1742.515, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1200.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1300.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 2400.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1200.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 2500.0, + "min": 1200.0, + "max": 3700.0 + }, + { + "current": 1200.0, + "min": 1200.0, + "max": 3700.0 + } + ], + "disk": { + "total": 467.8895797729492, + "used": 42.768035888671875 + }, + "gpu": "NVIDIA GeForce GTX TITAN X", + "gpu_count": 3, + "gpu_devices": [ + { + "name": "NVIDIA GeForce GTX TITAN X", + "memory_total": 12884901888 + }, + { + "name": "NVIDIA TITAN X (Pascal)", + "memory_total": 12884901888 + }, + { + "name": "NVIDIA GeForce GTX TITAN X", + "memory_total": 12884901888 + } + ], + "memory": { + "total": 31.240768432617188 + } +} diff --git a/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json b/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1651fe76da982bfd722f1ca26f465090dd633d78 --- /dev/null +++ b/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json @@ -0,0 +1 @@ +{"train_loss": 0.01587790111079812, "_timestamp": 1689179723.0025768, "_runtime": 45134.91465091705, "_step": 3000, "validation": {"_type": "images/separated", "width": 512, "height": 512, "format": "png", "count": 4, "filenames": ["media/images/validation_3000_88582cfed2f573568e58.png", "media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png", "media/images/validation_3000_400a50fac321ed4c2979.png", "media/images/validation_3000_72521f709e19fb5db039.png"], "captions": ["0: tom cruise thumbs up", "1: tom cruise thumbs up", "2: tom cruise thumbs up", "3: tom cruise thumbs up"]}, "_wandb": {"runtime": 45156}} \ No newline at end of file diff --git a/wandb/run-20230712_000308-yuype6tc/logs/debug-internal.log b/wandb/run-20230712_000308-yuype6tc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6464d9356fb67e4efb88916b04f4d38594c5db02 --- /dev/null +++ b/wandb/run-20230712_000308-yuype6tc/logs/debug-internal.log @@ -0,0 +1,46257 @@ +2023-07-12 00:03:08,088 INFO StreamThr :3148485 [internal.py:wandb_internal():86] W&B internal server running at pid: 3148485, started at: 2023-07-12 00:03:08.087197 +2023-07-12 00:03:08,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status +2023-07-12 00:03:08,092 INFO WriterThread:3148485 [datastore.py:open_for_write():85] open: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb +2023-07-12 00:03:08,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: header +2023-07-12 00:03:08,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: run +2023-07-12 00:03:08,316 INFO SenderThread:3148485 [dir_watcher.py:__init__():211] watching files in: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files +2023-07-12 00:03:08,317 INFO SenderThread:3148485 [sender.py:_start_run_threads():1100] run started: yuype6tc with start time 1689134588.087926 +2023-07-12 00:03:08,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:03:08,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: check_version +2023-07-12 00:03:08,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:03:08,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: check_version +2023-07-12 00:03:08,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: run_start +2023-07-12 00:03:08,457 DEBUG HandlerThread:3148485 [system_info.py:__init__():31] System info init +2023-07-12 00:03:08,457 DEBUG HandlerThread:3148485 [system_info.py:__init__():46] System info init done +2023-07-12 00:03:08,457 INFO HandlerThread:3148485 [system_monitor.py:start():181] Starting system monitor +2023-07-12 00:03:08,457 INFO SystemMonitor:3148485 [system_monitor.py:_start():145] Starting system asset monitoring threads +2023-07-12 00:03:08,459 INFO HandlerThread:3148485 [system_monitor.py:probe():201] Collecting system info +2023-07-12 00:03:08,459 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started cpu monitoring +2023-07-12 00:03:08,460 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started disk monitoring +2023-07-12 00:03:08,462 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started gpu monitoring +2023-07-12 00:03:08,462 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started memory monitoring +2023-07-12 00:03:08,463 INFO SystemMonitor:3148485 [interfaces.py:start():190] Started network monitoring +2023-07-12 00:03:08,519 DEBUG HandlerThread:3148485 [system_info.py:probe():195] Probing system +2023-07-12 00:03:08,528 DEBUG HandlerThread:3148485 [gitlib.py:_init_repo():53] git repository is invalid +2023-07-12 00:03:08,528 DEBUG HandlerThread:3148485 [system_info.py:probe():240] Probing system done +2023-07-12 00:03:08,529 DEBUG HandlerThread:3148485 [system_monitor.py:probe():210] {'os': 'Linux-5.15.0-60-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2023-07-12T04:03:08.519507', 'startedAt': '2023-07-12T04:03:08.061688', 'docker': None, 'cuda': None, 'args': ('--pretrained_model_name_or_path=runwayml/stable-diffusion-v1-5', '--train_data_dir=/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/', '--resolution=512', '--center_crop', '--random_flip', '--train_batch_size=2', '--gradient_accumulation_steps=4', '--num_train_epochs=300', '--learning_rate=1e-5', '--max_grad_norm=1', '--lr_scheduler=cosine', '--lr_warmup_steps=500', '--output_dir=/l/vision/v5/sragas/easel_ai/models_plain/', '--checkpointing_steps=500', '--report_to=wandb', '--validation_prompt=tom cruise thumbs up', '--seed=51', '--push_to_hub', '--hub_model_id=person-thumbs-up-plain-lora'), 'state': 'running', 'program': 'train_text_to_image_lora.py', 'codePath': 'train_text_to_image_lora.py', 'host': 'magneton', 'username': 'sragas', 'executable': '/nfs/blitzle/home/data/vision5/sragas/easel_venv/bin/python3', 'cpu_count': 6, 'cpu_count_logical': 12, 'cpu_freq': {'current': 1779.2625, 'min': 1200.0, 'max': 3700.0}, 'cpu_freq_per_core': [{'current': 3100.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1500.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}, {'current': 2100.0, 'min': 1200.0, 'max': 3700.0}, {'current': 3100.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1742.515, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1300.0, 'min': 1200.0, 'max': 3700.0}, {'current': 2400.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}, {'current': 2500.0, 'min': 1200.0, 'max': 3700.0}, {'current': 1200.0, 'min': 1200.0, 'max': 3700.0}], 'disk': {'total': 467.8895797729492, 'used': 42.768035888671875}, 'gpu': 'NVIDIA GeForce GTX TITAN X', 'gpu_count': 3, 'gpu_devices': [{'name': 'NVIDIA GeForce GTX TITAN X', 'memory_total': 12884901888}, {'name': 'NVIDIA TITAN X (Pascal)', 'memory_total': 12884901888}, {'name': 'NVIDIA GeForce GTX TITAN X', 'memory_total': 12884901888}], 'memory': {'total': 31.240768432617188}} +2023-07-12 00:03:08,529 INFO HandlerThread:3148485 [system_monitor.py:probe():211] Finished collecting system info +2023-07-12 00:03:08,529 INFO HandlerThread:3148485 [system_monitor.py:probe():214] Publishing system info +2023-07-12 00:03:08,529 DEBUG HandlerThread:3148485 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2023-07-12 00:03:08,540 DEBUG HandlerThread:3148485 [system_info.py:_save_pip():67] Saving pip packages done +2023-07-12 00:03:08,542 INFO HandlerThread:3148485 [system_monitor.py:probe():216] Finished publishing system info +2023-07-12 00:03:08,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:03:08,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now +2023-07-12 00:03:08,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:08,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:08,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: telemetry +2023-07-12 00:03:09,042 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s7kvtbzx-wandb-metadata.json +2023-07-12 00:03:09,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/requirements.txt +2023-07-12 00:03:09,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:09,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:03:09,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json +2023-07-12 00:03:11,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:13,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:13,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:17,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:18,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: telemetry +2023-07-12 00:03:18,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:18,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: config +2023-07-12 00:03:21,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:23,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:23,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:24,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:27,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:29,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:30,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:31,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:32,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:03:33,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:35,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:35,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:37,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:38,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:38,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:41,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:41,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:03:41,229 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:03:41,229 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:03:41,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:03:41,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:03:41,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/config.yaml +2023-07-12 00:03:41,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:43,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:45,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:46,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:47,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:49,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:50,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:03:50,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:03:50,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:03:50,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:03:51,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:03:51,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:51,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:53,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:03:53,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:03:55,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:57,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:03:57,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:03:59,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:00,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:00,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:00,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:00,190 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:00,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:01,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:03,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:03,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:05,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:08,464 DEBUG SystemMonitor:3148485 [system_monitor.py:_start():159] Starting system metrics aggregation loop +2023-07-12 00:04:08,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:04:08,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:08,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:08,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:09,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:09,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:09,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:09,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:09,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:10,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:11,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:13,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:13,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:17,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:19,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:19,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:20,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:20,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:20,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:20,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:21,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:21,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:23,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:23,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:24,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:25,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:27,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:29,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:29,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:29,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:29,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:29,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:30,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:30,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:31,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:33,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:36,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:37,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:38,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:04:38,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:38,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:39,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:40,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:40,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:40,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:40,804 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:41,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:41,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:41,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:43,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:45,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:47,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:47,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:49,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:50,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:50,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:50,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:50,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:51,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:51,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:52,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:53,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:54,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:04:54,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:04:54,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:04:54,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:04:54,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:04:54,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:04:55,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:04:55,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:57,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:04:58,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:04:59,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:05:03,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:08,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:08,468 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:05:09,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:09,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:13,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:18,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:23,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:24,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:24,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:29,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:34,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:38,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:05:39,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:39,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:39,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:44,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:49,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:05:54,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:05:54,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:05:55,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:00,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:03,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:03,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:06:03,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media +2023-07-12 00:06:03,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_0838dad447e339fd0fea.png with policy now +2023-07-12 00:06:03,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_a8dfbe0ff96ea5050767.png with policy now +2023-07-12 00:06:03,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,687 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_56c2e575b2b9d92db6f0.png with policy now +2023-07-12 00:06:03,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:06:03,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_10_3f8ba95016c3d0286ac3.png with policy now +2023-07-12 00:06:03,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:03,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:03,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:03,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:03,949 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zmy0twhm-media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:04,033 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3qduvr0-media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 00:06:04,048 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1gdybl44-media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 00:06:04,093 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yfkhyw3b-media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 00:06:04,503 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s4k0p976-media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:04,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:04,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:04,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 00:06:04,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 00:06:04,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 00:06:04,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:06:05,002 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lft3007p-media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 00:06:05,004 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/db4zocz3-media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 00:06:05,036 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fs39n25q-media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 00:06:05,242 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xmns3160-media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 00:06:05,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:07,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:08,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:06:09,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:10,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:10,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:10,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:11,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:12,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:15,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:15,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:17,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:19,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:21,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:21,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:23,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:23,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:23,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:23,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:23,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:25,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:25,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:25,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:27,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:27,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:29,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:31,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:32,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:32,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:32,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:32,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:32,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:32,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:33,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:35,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:37,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:38,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:06:39,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:40,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:40,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:41,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:41,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:41,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:41,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:41,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:41,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:43,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:43,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:45,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:47,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:48,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:49,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:51,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:06:51,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:06:51,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:06:51,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:06:51,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:06:53,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:53,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:06:55,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:55,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:06:55,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:06:57,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:06:59,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:01,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:01,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:01,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:01,471 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:01,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:01,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:03,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:04,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:05,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:07,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:08,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:07:09,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:10,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:10,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:10,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:07:11,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:11,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:11,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:11,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:11,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:13,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:15,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:16,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:17,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:21,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:21,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:21,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:21,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:21,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:21,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:22,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:23,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:25,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:25,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:25,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:07:27,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:28,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:29,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:30,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:30,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:30,983 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:30,985 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:31,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:31,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:33,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:35,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:07:35,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:07:35,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:07:35,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:07:35,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:07:35,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:37,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:07:38,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:07:39,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:40,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:40,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:07:45,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:50,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:55,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:07:55,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:07:55,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:00,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:05,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:08,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:08:10,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:10,978 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:11,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:16,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:21,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:25,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:25,978 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:26,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:29,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_b1104143852d0cdcfdbc.png with policy now +2023-07-12 00:08:29,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_a492b392b73e859b8a77.png with policy now +2023-07-12 00:08:29,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_3ac29936b57bab3b60df.png with policy now +2023-07-12 00:08:29,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:08:29,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_20_0394766c00f144b38aaf.png with policy now +2023-07-12 00:08:30,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:30,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:08:30,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:08:30,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:08:30,257 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4tjmg87-media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 00:08:30,314 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j0fggkff-media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 00:08:30,316 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sm8pmsmk-media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 00:08:30,322 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o51jb73i-media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 00:08:30,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:08:30,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 00:08:30,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 00:08:30,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 00:08:30,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 00:08:30,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:08:31,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:31,262 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qq0fimgp-media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 00:08:31,263 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q2cpsx7h-media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 00:08:31,285 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9qsl438y-media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 00:08:31,292 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6263jl6v-media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 00:08:33,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:36,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:37,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:38,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:08:39,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:39,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:41,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:41,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:41,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:43,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:45,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:46,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:47,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:49,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:49,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:08:49,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:08:49,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:08:49,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:08:51,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:51,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:53,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:55,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:08:56,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:08:56,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:08:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:08:59,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:08:59,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:08:59,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:08:59,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:08:59,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:08:59,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:01,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:01,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:03,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:05,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:07,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:07,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:08,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:09:09,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:09,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:09,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:09,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:09,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:11,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:11,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:11,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:13,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:13,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:16,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:19,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:19,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:19,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:19,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:19,839 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:20,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:20,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:22,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:24,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:24,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:26,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:26,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:26,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:29,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:29,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:29,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:29,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:29,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:30,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:30,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:32,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:34,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:34,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:36,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:38,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:09:39,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:39,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:39,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:39,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:39,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:40,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:40,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:41,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:41,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:42,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:44,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:45,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:46,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:50,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:50,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:09:50,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:50,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:09:50,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:09:50,600 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:09:51,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:09:52,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:56,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:09:56,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:09:56,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:09:56,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:09:58,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:00,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:01,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:10:01,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:10:01,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:10:01,198 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:10:02,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:10:02,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:02,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:05,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:10:05,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:10:05,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:10:05,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:10:06,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:10:06,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:07,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:08,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:10:08,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:10:11,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:11,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:13,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:18,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:23,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:26,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:26,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:28,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:33,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:38,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:10:39,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:42,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:42,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:45,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:50,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:55,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:10:57,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:10:57,279 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:10:59,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_ae28d8db362e0aa9f573.png with policy now +2023-07-12 00:10:59,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_616207d75a28a906c55b.png with policy now +2023-07-12 00:10:59,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_2c0d6dfc2d246283392e.png with policy now +2023-07-12 00:10:59,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:10:59,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_30_ed7f0d8809d791b958b4.png with policy now +2023-07-12 00:10:59,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:10:59,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:10:59,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:10:59,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:00,079 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e2ufjigs-media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 00:11:00,120 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6dvjs6kd-media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 00:11:00,121 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wsb7bsl6-media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 00:11:00,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:00,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 00:11:00,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 00:11:00,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 00:11:00,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 00:11:00,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:11:00,254 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sc1ybcfp-media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 00:11:00,524 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lnva8x1k-media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 00:11:00,538 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9qhfu4uq-media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 00:11:00,545 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q8h0813r-media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 00:11:00,644 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0l8safbx-media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 00:11:00,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:04,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:05,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:06,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:08,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:08,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:11:09,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:11,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:12,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:12,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:12,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:14,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:16,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:17,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:20,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:20,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:20,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:20,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:20,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:21,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:22,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:22,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:24,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:26,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:27,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:27,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:27,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:30,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:30,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:30,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:30,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:30,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:30,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:32,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:33,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:34,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:38,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:38,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:11:38,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:40,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:40,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:40,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:40,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:40,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:41,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:42,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:42,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:42,665 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:44,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:44,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:48,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:49,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:50,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:51,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:11:51,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:11:51,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:11:51,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:11:52,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:11:54,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:55,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:11:56,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:11:57,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:11:57,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:11:58,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:00,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:00,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:01,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:01,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:01,866 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:01,868 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:02,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:04,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:05,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:06,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:08,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:08,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:12:10,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:11,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:11,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:11,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:11,482 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:11,484 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:12,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:13,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:13,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:12:14,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:16,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:16,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:18,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:20,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:21,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:21,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:21,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:21,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:21,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:22,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:24,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:26,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:26,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:28,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:28,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:28,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:12:30,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:31,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:31,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:31,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:31,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:32,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:32,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:32,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:36,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:36,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:12:36,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:12:36,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:12:36,650 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:12:37,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:12:38,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:38,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:12:38,487 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:12:43,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:43,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:43,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:12:48,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:53,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:58,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:12:59,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:12:59,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:04,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:08,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:13:09,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:14,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:14,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:15,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:20,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:25,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:29,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:29,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:30,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:30,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_beffc069b9ded929df09.png with policy now +2023-07-12 00:13:30,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,882 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_83474bbbae95371e4fe1.png with policy now +2023-07-12 00:13:30,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_8ede4c40d45be37a3114.png with policy now +2023-07-12 00:13:30,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:13:30,903 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_40_cb5f34fc925cd4802446.png with policy now +2023-07-12 00:13:30,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:30,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:13:30,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:13:30,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:13:31,159 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fnjomyyl-media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 00:13:31,211 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/japiy359-media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 00:13:31,221 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zjf1c6ic-media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 00:13:31,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e57km2oq-media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 00:13:31,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:13:31,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 00:13:31,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 00:13:31,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 00:13:31,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 00:13:31,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:13:31,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nffzyvga-media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 00:13:31,895 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5w5ak3yy-media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 00:13:31,907 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e1yflci4-media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 00:13:31,981 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ns9mxiuz-media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 00:13:35,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:36,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:38,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:13:38,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:40,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:41,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:41,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:42,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:44,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:44,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:46,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:46,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:48,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:50,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:50,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:13:50,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:13:50,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:13:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:13:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:52,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:52,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:54,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:56,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:57,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:13:58,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:13:59,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:13:59,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:13:59,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:13:59,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:13:59,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:13:59,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:00,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:00,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:02,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:03,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:04,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:06,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:08,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:14:08,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:08,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:09,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:09,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:09,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:09,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:09,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:10,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:13,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:14,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:14:14,460 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:14:14,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:16,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:18,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:19,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:20,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:20,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:20,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:20,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:20,589 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:22,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:24,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:25,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:26,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:29,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:14:29,460 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:14:30,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:30,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:30,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:30,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:30,746 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:30,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:31,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:32,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:34,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:35,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:38,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:14:38,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:40,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:40,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:40,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:40,690 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:40,692 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:41,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:41,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:42,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:44,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:45,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:14:45,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:14:47,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:48,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:50,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:51,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:14:51,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:14:51,306 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:14:51,308 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:14:51,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:14:52,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:52,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:56,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:14:58,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:14:58,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:00,674 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:01,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:01,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:02,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:15:02,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:15:02,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:15:02,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:15:02,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:15:04,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:04,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:06,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:15:06,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:15:06,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:15:06,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:15:06,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:15:06,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:08,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:15:08,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:15:09,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:14,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:16,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:16,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:19,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:24,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:29,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:31,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:31,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:35,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:38,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:15:41,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:46,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:46,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:15:46,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:15:51,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:15:56,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:00,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_6c49012791de94816289.png with policy now +2023-07-12 00:16:00,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,676 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_5fc1a953a335a81d0538.png with policy now +2023-07-12 00:16:00,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_f43b6d21c133e2a6b003.png with policy now +2023-07-12 00:16:00,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:16:00,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_50_8bd9326976928e17b3ca.png with policy now +2023-07-12 00:16:00,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:00,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:00,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:00,738 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:00,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:00,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_6c49012791de94816289.png +2023-07-12 00:16:00,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 00:16:00,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 00:16:00,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 00:16:00,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:16:00,974 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/42ait1o9-media/images/validation_50_6c49012791de94816289.png +2023-07-12 00:16:01,003 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ef1s45rv-media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 00:16:01,006 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1pe31w2-media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 00:16:01,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mv7jnge0-media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 00:16:01,382 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9r8deach-media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 00:16:01,426 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9a0n3hwt-media/images/validation_50_6c49012791de94816289.png +2023-07-12 00:16:01,688 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rnmo2lgf-media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 00:16:01,766 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6q92dm9z-media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 00:16:01,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:01,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:02,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:06,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:08,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:08,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:16:08,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:10,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:11,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:12,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:13,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:14,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:16,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:17,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:17,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:18,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:20,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:20,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:20,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:20,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:20,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:20,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:22,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:24,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:24,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:28,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:29,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:30,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:31,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:31,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:31,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:31,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:31,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:32,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:33,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:33,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:34,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:36,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:38,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:16:38,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:40,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:40,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:40,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:40,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:40,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:40,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:41,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:42,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:44,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:45,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:46,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:48,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:16:48,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:16:50,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:50,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:51,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:16:51,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:16:51,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:16:51,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:16:51,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:16:52,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:54,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:16:56,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:16:56,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:00,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:00,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:00,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:00,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:00,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:00,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:02,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:02,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:03,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:03,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:04,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:07,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:08,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:17:08,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:10,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:10,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:10,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:10,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:10,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:10,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:12,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:13,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:14,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:16,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:18,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:18,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:18,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:20,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:20,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:20,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:20,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:20,996 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:21,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:22,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:23,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:24,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:28,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:29,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:30,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:31,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:31,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:31,490 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:31,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:31,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:32,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:33,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:33,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:35,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:36,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:17:36,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:17:36,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:17:36,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:17:37,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:17:37,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:38,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:17:39,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:17:40,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:45,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:48,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:17:48,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:17:50,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:17:55,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:00,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:03,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:03,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:06,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:08,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:18:12,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:17,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:18,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:18,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:22,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:27,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:30,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_6ae01c6a6a2bfa713e44.png with policy now +2023-07-12 00:18:30,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_cfabe222535005b7c73d.png with policy now +2023-07-12 00:18:30,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_ddf1c39281d80912060e.png with policy now +2023-07-12 00:18:30,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:18:30,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_60_ed363dbde584f9cd8a96.png with policy now +2023-07-12 00:18:30,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:30,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:18:30,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:18:30,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:18:31,007 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q5g873z9-media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 00:18:31,013 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6duhdnk9-media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 00:18:31,019 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ubg9twvg-media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 00:18:31,030 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/73miy2qg-media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 00:18:31,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:18:31,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 00:18:31,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 00:18:31,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 00:18:31,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 00:18:31,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:18:31,453 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/19aqlk2v-media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 00:18:31,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5x31q3in-media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 00:18:31,468 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8zx1d2g4-media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 00:18:31,620 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlt62v6a-media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 00:18:33,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:33,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:33,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:35,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:37,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:38,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:18:38,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:40,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:41,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:43,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:43,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:45,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:47,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:48,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:18:48,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:48,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:18:49,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:49,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:49,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:18:49,427 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:18:49,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:18:50,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:18:51,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:53,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:54,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:57,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:59,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:18:59,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:18:59,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:18:59,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:18:59,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:18:59,717 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:00,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:01,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:03,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:04,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:04,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:04,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:07,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:08,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:19:09,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:10,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:10,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:10,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:10,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:10,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:11,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:13,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:15,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:16,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:17,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:19,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:19,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:19,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:20,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:20,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:20,562 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:20,564 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:21,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:21,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:23,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:25,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:27,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:27,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:29,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:30,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:30,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:30,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:30,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:31,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:32,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:33,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:34,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:34,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:35,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:37,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:38,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:19:38,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:39,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:40,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:40,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:40,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:40,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:41,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:43,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:43,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:45,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:47,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:49,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:50,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:19:50,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:19:51,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:19:51,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:19:51,094 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:19:51,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:19:51,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:19:51,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:53,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:54,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:19:55,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:57,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:19:59,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:01,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:20:01,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:20:01,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:20:01,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:20:01,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:20:01,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:03,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:05,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:05,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:05,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:05,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:05,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:20:05,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:20:05,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:20:05,609 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:20:06,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:20:07,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:08,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:20:09,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:20:10,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:15,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:20,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:20,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:20,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:25,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:30,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:35,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:35,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:36,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:38,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:20:42,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:47,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:50,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:20:50,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:20:52,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:20:57,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:00,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_09c22363614ab958c3bc.png with policy now +2023-07-12 00:21:00,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_eba1f69e2b11ef8f44d3.png with policy now +2023-07-12 00:21:00,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_d01dfe2d2aeaa1bec479.png with policy now +2023-07-12 00:21:00,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:21:00,359 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_70_24254896e341a078f630.png with policy now +2023-07-12 00:21:00,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:00,404 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:00,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:00,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:00,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:00,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 00:21:00,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_24254896e341a078f630.png +2023-07-12 00:21:00,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 00:21:00,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 00:21:00,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:21:00,657 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9gqvcafo-media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 00:21:00,681 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ysk754mc-media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 00:21:00,702 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bjb6u0r3-media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 00:21:00,709 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/woh0j4tr-media/images/validation_70_24254896e341a078f630.png +2023-07-12 00:21:01,050 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qphexw0y-media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 00:21:01,085 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8q4mwhlm-media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 00:21:01,091 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y6jl2dns-media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 00:21:01,400 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kpm4w6i7-media/images/validation_70_24254896e341a078f630.png +2023-07-12 00:21:03,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:04,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:05,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:05,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:06,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:08,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:08,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:21:08,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:10,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:12,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:13,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:14,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:16,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:18,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:19,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:19,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:19,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:19,581 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:19,583 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:20,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:20,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:20,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:20,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:22,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:24,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:24,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:28,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:29,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:29,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:29,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:29,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:29,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:29,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:30,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:32,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:34,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:36,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:36,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:36,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:38,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:21:38,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:40,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:40,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:40,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:40,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:40,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:40,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:40,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:44,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:45,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:46,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:48,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:50,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:21:50,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:21:50,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:21:50,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:21:50,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:21:50,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:51,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:51,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:21:51,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:21:54,564 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:56,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:21:56,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:21:58,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:00,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:00,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:00,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:00,503 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:00,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:01,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:02,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:04,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:06,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:06,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:06,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:06,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:08,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:22:08,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:11,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:11,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:11,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:11,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:11,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:12,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:12,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:14,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:17,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:18,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:20,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:21,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:21,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:21,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:21,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:21,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:21,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:21,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:22,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:22,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:24,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:28,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:28,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:30,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:31,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:31,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:31,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:31,391 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:31,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:32,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:34,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:34,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:36,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:22:36,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:22:36,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:22:36,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:22:36,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:22:36,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:36,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:36,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:38,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:22:38,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:22:39,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:44,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:49,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:22:52,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:22:52,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:22:55,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:00,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:05,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:07,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:07,028 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:08,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:23:10,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:15,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:20,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:22,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:22,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:26,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:30,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_64b2c49cff2d363d20ad.png with policy now +2023-07-12 00:23:30,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_f3e5374d59497f9be535.png with policy now +2023-07-12 00:23:30,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_653670180f2536edf28e.png with policy now +2023-07-12 00:23:30,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:23:30,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_80_ca894d05d078201593b2.png with policy now +2023-07-12 00:23:30,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:30,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:23:30,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:23:30,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:23:30,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:23:30,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 00:23:30,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 00:23:30,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_653670180f2536edf28e.png +2023-07-12 00:23:30,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 00:23:30,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:23:30,888 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/odczyus8-media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 00:23:30,949 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ebgh994-media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 00:23:30,950 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zduxw7n-media/images/validation_80_653670180f2536edf28e.png +2023-07-12 00:23:30,978 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nrir2twy-media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 00:23:31,297 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/evgcmjsm-media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 00:23:31,299 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fnlh6iys-media/images/validation_80_653670180f2536edf28e.png +2023-07-12 00:23:31,376 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7yy9cotf-media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 00:23:31,627 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mceu715a-media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 00:23:31,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:34,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:36,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:36,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:37,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:37,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:38,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:23:38,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:40,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:40,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:42,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:44,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:46,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:47,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:48,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:49,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:49,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:23:49,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:23:49,942 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:23:50,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:23:50,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:52,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:23:52,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:23:52,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:53,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:54,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:58,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:23:58,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:23:59,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:23:59,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:23:59,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:23:59,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:23:59,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:00,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:02,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:04,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:06,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:07,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:07,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:08,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:24:08,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:09,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:09,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:09,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:09,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:09,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:09,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:10,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:12,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:15,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:16,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:18,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:20,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:20,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:20,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:20,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:20,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:20,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:20,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:22,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:22,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:24,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:25,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:28,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:29,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:29,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:29,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:29,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:30,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:30,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:30,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:34,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:36,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:36,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:37,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:37,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:38,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:24:38,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:40,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:40,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:40,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:40,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:41,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:41,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:42,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:44,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:46,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:47,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:50,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:51,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:24:51,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:24:51,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:24:51,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:24:51,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:24:52,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:52,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:53,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:24:53,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:24:54,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:24:58,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:24:58,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:01,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:02,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:25:02,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:25:02,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:25:02,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:25:03,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:25:03,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:04,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:05,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:06,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:25:06,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:25:06,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:25:06,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:25:07,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:25:07,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:08,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:25:08,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:08,781 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:09,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:25:09,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:14,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:19,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:23,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:23,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:24,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:29,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:35,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:38,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:25:38,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:38,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:40,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:45,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:50,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:25:53,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:25:53,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:25:55,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:00,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,507 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_d4b9dc3d41c12fcb9fe7.png with policy now +2023-07-12 00:26:00,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_21a56bd8f7d955ce07a3.png with policy now +2023-07-12 00:26:00,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_8b813df4910f6e14fe9f.png with policy now +2023-07-12 00:26:00,551 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:26:00,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_90_b276db68b1c2650b37f5.png with policy now +2023-07-12 00:26:00,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:00,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:00,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:00,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:00,809 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s9933rxu-media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 00:26:00,870 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p6brayzl-media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 00:26:00,918 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7t84wvff-media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 00:26:00,922 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dvwh5op3-media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 00:26:01,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:01,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 00:26:01,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 00:26:01,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 00:26:01,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 00:26:01,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:26:01,481 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ov1qe56-media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 00:26:01,482 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bx2foth0-media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 00:26:01,485 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o0rzi9o4-media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 00:26:01,504 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ml0som78-media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 00:26:01,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:05,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:07,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:07,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:08,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:26:09,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:09,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:11,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:11,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:13,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:13,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:15,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:17,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:18,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:19,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:19,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:19,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:19,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:19,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:20,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:21,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:23,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:24,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:24,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:24,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:25,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:29,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:29,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:29,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:29,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:29,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:29,453 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:30,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:31,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:33,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:34,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:37,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:38,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:26:39,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:39,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:39,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:39,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:40,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:40,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:40,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:40,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:41,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:41,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:43,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:44,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:47,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:49,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:49,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:49,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:49,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:49,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:26:50,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:26:50,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:51,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:53,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:54,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:26:54,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:26:55,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:26:57,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:59,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:26:59,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:26:59,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:26:59,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:26:59,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:00,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:00,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:01,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:03,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:06,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:07,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:08,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:27:09,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:09,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:09,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:27:10,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:10,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:10,086 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:10,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:10,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:11,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:12,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:13,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:17,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:17,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:19,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:20,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:20,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:20,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:20,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:21,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:21,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:22,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:25,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:25,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:27:25,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:27,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:28,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:29,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:31,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:31,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:31,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:31,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:31,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:33,384 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:33,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:35,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:35,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:27:35,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:27:35,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:27:35,670 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:27:36,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:27:37,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:27:38,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:27:38,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:40,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:40,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:27:44,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:49,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:54,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:27:55,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:27:55,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:00,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:05,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:08,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:28:10,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:10,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:10,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:15,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:20,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:25,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:25,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:25,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:29,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_661b1a72c771caa0e001.png with policy now +2023-07-12 00:28:29,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_85a79c24055b252b391d.png with policy now +2023-07-12 00:28:29,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_e3b8328c3efd11dc0868.png with policy now +2023-07-12 00:28:29,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:28:29,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_100_5f675bc00f07f97610c2.png with policy now +2023-07-12 00:28:30,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:30,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:28:30,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:28:30,023 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:28:30,255 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bp294jcq-media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 00:28:30,286 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ntvbmgn2-media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 00:28:30,315 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nqyst2kx-media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 00:28:30,325 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bad3jhhc-media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 00:28:30,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:28:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 00:28:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 00:28:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 00:28:30,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 00:28:30,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:28:30,860 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/58kmyqby-media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 00:28:30,866 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iuwy4cyu-media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 00:28:30,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oyf8n66j-media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 00:28:30,877 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/15hkeu2g-media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 00:28:31,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:33,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:36,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:37,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:38,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:28:39,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:40,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:40,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:40,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:41,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:41,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:43,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:47,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:47,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:49,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:49,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:49,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:28:49,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:28:49,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:28:50,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:28:51,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:52,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:53,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:55,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:28:55,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:28:55,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:57,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:28:58,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:28:59,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:28:59,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:28:59,180 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:28:59,182 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:28:59,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:01,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:03,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:04,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:05,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:07,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:08,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:29:09,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:09,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:09,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:09,778 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:09,780 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:10,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:10,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:10,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:11,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:13,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:14,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:15,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:19,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:20,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:20,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:20,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:20,147 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:20,149 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:20,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:21,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:25,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:25,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:25,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:25,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:27,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:29,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:30,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:31,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:31,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:31,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:31,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:31,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:33,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:35,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:35,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:37,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:38,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:29:39,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:40,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:40,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:40,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:41,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:41,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:41,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:41,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:41,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:43,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:45,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:45,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:47,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:49,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:51,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:29:51,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:51,488 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:29:51,488 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:29:51,490 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:29:51,752 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:29:53,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:55,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:29:55,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:29:55,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:57,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:29:57,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:29:59,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:01,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:30:01,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:30:01,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:30:01,053 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:30:01,780 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:30:03,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:03,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:05,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:30:05,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:30:05,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:30:05,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:30:05,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:30:05,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:07,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:30:08,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:08,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:30:10,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:10,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:30:13,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:18,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:23,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:25,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:25,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:30:29,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:34,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:38,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:30:40,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:40,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:40,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:30:46,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:51,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:56,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:30:56,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:30:56,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:00,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,647 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_ed89fce5f94211880d08.png with policy now +2023-07-12 00:31:00,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_519be1aabb2ce5adb552.png with policy now +2023-07-12 00:31:00,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_0a38067c3b4b0763814d.png with policy now +2023-07-12 00:31:00,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:31:00,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_110_57b921c4856dfbee5311.png with policy now +2023-07-12 00:31:00,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:00,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:00,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:00,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 00:31:00,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 00:31:00,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 00:31:00,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:31:01,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3uw9s60a-media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 00:31:01,050 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/671lna2s-media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 00:31:01,057 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oiczommb-media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 00:31:01,118 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i7iaks7w-media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 00:31:01,456 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q8dchois-media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 00:31:01,641 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1gphfzhd-media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 00:31:01,673 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cghivl3s-media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 00:31:01,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:01,859 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/96lzp3r5-media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 00:31:05,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:07,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:07,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:08,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:31:11,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:11,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:11,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:11,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:13,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:13,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:15,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:17,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:19,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:21,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:21,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:21,739 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:21,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:21,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:21,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:23,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:25,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:25,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:26,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:26,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:27,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:29,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:30,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:31,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:31,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:31,028 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:31,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:31,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:31,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:35,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:36,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:37,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:38,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:31:40,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:41,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:41,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:41,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:41,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:41,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:41,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:41,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:42,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:44,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:46,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:46,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:48,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:51,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:31:51,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:51,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:31:51,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:31:51,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:31:52,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:31:52,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:54,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:56,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:31:56,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:31:56,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:31:57,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:31:58,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:02,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:02,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:02,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:02,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:02,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:02,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:03,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:04,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:06,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:08,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:08,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:32:08,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:11,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:11,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:11,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:11,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:11,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:11,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:12,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:12,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:13,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:14,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:16,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:18,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:19,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:22,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:22,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:22,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:22,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:22,212 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:23,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:24,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:25,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:26,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:26,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:26,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:30,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:30,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:32,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:32,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:32,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:32,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:32,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:33,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:34,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:36,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:36,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:37,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:32:37,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:32:37,309 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:32:37,311 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:32:38,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:32:38,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:32:40,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:32:41,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:41,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:41,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:46,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:51,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:32:56,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:32:56,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:32:56,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:01,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:06,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:08,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:33:12,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:12,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:12,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:17,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:22,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:27,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:27,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:27,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:31,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_b4dbdc5959d71b1b0010.png with policy now +2023-07-12 00:33:31,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_3aa26b32048f4df01e9f.png with policy now +2023-07-12 00:33:31,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_18e3cfcb4e5db9c7981b.png with policy now +2023-07-12 00:33:31,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:33:31,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_120_f3f35ebd04d7e55194ae.png with policy now +2023-07-12 00:33:31,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:33:31,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:33:31,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:33:31,944 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:33:32,262 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jntiff99-media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 00:33:32,303 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cr02ki2y-media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 00:33:32,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:33:32,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 00:33:32,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 00:33:32,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 00:33:32,315 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4f9piivd-media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 00:33:32,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 00:33:32,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:33:32,327 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vjyn8i9f-media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 00:33:32,839 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6e9gix8a-media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 00:33:32,922 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l167ysr6-media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 00:33:32,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:32,954 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bqknbxsp-media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 00:33:33,043 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cgq95ug3-media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 00:33:36,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:38,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:38,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:33:38,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:40,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:41,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:33:42,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:42,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:42,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:43,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:46,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:48,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:49,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:50,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:50,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:33:50,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:33:50,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:33:50,959 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:33:51,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:33:52,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:54,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:33:56,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:57,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:33:57,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:33:58,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:33:59,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:00,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:00,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:00,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:00,042 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:00,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:00,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:02,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:04,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:05,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:06,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:08,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:08,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:34:09,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:09,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:09,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:09,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:10,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:11,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:12,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:12,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:12,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:14,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:16,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:16,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:18,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:19,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:19,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:19,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:19,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:20,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:21,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:22,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:24,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:26,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:27,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:27,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:30,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:30,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:30,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:30,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:30,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:31,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:32,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:32,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:34,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:38,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:38,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:38,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:34:40,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:40,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:40,777 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:40,777 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:40,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:41,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:42,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:43,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:43,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:43,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:46,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:48,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:48,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:50,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:51,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:34:51,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:34:51,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:34:51,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:34:52,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:34:54,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:34:54,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:56,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:58,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:34:58,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:34:58,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:34:59,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:00,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:02,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:35:02,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:35:02,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:35:02,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:35:02,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:35:04,582 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:04,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:06,589 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:06,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:35:06,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:35:06,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:35:06,844 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:35:07,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:35:08,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:35:08,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:35:10,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:13,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:13,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:35:15,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:20,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:25,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:28,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:28,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:35:30,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:35,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:38,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:35:41,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:43,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:43,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:35:46,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:51,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:56,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:35:58,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:35:58,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:01,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,241 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_3e5c971e8f903ffb6755.png with policy now +2023-07-12 00:36:01,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_b27cf42472a6e5022d7a.png with policy now +2023-07-12 00:36:01,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_db9b8ff059ac75e60033.png with policy now +2023-07-12 00:36:01,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:36:01,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_130_1846f2582dc6add0423d.png with policy now +2023-07-12 00:36:01,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:01,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:01,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:01,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:01,584 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0inophzk-media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 00:36:01,687 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dkai2j6d-media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 00:36:01,703 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iveoso0e-media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 00:36:01,704 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvceg8e5-media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 00:36:01,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:01,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 00:36:01,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 00:36:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 00:36:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 00:36:01,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:36:02,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:02,412 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xz0731oq-media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 00:36:02,413 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d04m3zfu-media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 00:36:02,416 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sljtltlp-media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 00:36:02,456 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vl09kwjy-media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 00:36:06,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:08,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:08,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:36:08,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:10,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:10,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:12,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:13,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:13,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:13,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:14,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:18,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:19,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:20,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:20,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:20,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:20,243 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:20,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:20,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:22,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:24,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:24,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:26,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:29,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:29,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:29,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:30,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:30,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:30,102 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:30,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:30,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:30,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:32,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:34,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:34,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:36,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:38,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:36:38,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:40,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:40,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:40,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:40,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:40,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:40,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:42,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:44,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:44,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:36:44,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:46,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:46,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:50,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:36:50,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:36:50,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:36:50,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:36:50,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:36:50,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:51,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:52,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:54,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:56,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:36:57,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:36:59,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:36:59,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:00,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:00,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:00,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:00,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:00,898 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:00,898 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:02,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:02,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:04,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:07,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:08,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:37:08,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:10,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:11,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:11,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:11,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:11,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:11,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:12,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:13,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:14,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:14,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:16,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:18,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:18,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:20,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:21,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:21,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:21,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:21,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:21,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:22,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:23,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:26,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:28,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:28,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:29,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:29,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:30,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:31,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:31,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:31,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:31,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:31,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:34,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:35,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:37:36,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:37:36,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:37:36,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:37:37,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:37:37,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:38,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:37:39,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:37:39,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:44,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:44,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:44,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:49,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:54,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:37:59,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:37:59,625 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:37:59,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:04,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:08,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:38:10,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:14,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:38:14,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:38:15,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:20,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:25,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:29,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:38:29,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:38:30,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_20276ffa7780c15c8826.png with policy now +2023-07-12 00:38:30,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_5f10192137f91e3d88e7.png with policy now +2023-07-12 00:38:30,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_f11466fe48b33d6f9da9.png with policy now +2023-07-12 00:38:30,675 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:38:30,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_140_8cf36f31bd00d9abfd34.png with policy now +2023-07-12 00:38:30,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:38:30,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:38:30,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:38:30,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:30,709 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:38:30,943 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m5yivnxi-media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 00:38:30,987 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/erf6mck5-media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 00:38:31,001 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3obh3r2x-media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 00:38:31,003 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/di5yinl7-media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 00:38:31,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:38:31,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 00:38:31,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 00:38:31,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 00:38:31,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 00:38:31,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:38:31,658 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xgahyfh8-media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 00:38:31,716 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z0ip3jvj-media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 00:38:31,721 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ialsxms9-media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 00:38:31,744 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gcqqedml-media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 00:38:35,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:36,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:38,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:38,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:38:40,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:40,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:38:41,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:42,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:44,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:45,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:38:45,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:38:46,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:47,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:50,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:50,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:38:50,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:38:50,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:38:50,552 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:38:51,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:38:52,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:52,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:38:54,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:56,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:38:58,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:00,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:00,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:00,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:00,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:00,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:00,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:00,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:01,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:02,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:03,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:04,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:06,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:08,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:39:08,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:10,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:10,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:10,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:10,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:10,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:11,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:12,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:13,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:14,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:15,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:15,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:16,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:19,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:20,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:21,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:21,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:21,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:21,117 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:21,315 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:22,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:24,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:24,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:28,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:30,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:30,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:31,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:31,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:31,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:31,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:31,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:31,956 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:32,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:34,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:35,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:36,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:38,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:38,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:39:40,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:40,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:41,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:41,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:41,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:41,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:42,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:42,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:45,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:46,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:46,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:39:46,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:39:48,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:51,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:52,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:52,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:39:52,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:39:52,604 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:39:52,606 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:39:53,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:39:54,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:56,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:39:57,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:39:58,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:02,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:02,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:02,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:02,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:02,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:40:02,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:40:02,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:40:02,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:40:03,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:40:04,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:06,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:07,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:07,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:40:07,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:40:07,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:40:07,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:40:07,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:40:08,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:08,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:40:10,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:40:12,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:17,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:17,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:18,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:23,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:28,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:32,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:32,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:34,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:38,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:40:39,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:44,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:47,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:40:47,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:40:50,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:40:55,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:00,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:01,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,804 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_ad78c61f0f7db6d206fd.png with policy now +2023-07-12 00:41:01,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_da52d8afd9d0f2869f66.png with policy now +2023-07-12 00:41:01,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,846 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_c4f6c02b99e6533a2eb2.png with policy now +2023-07-12 00:41:01,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:41:01,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_150_843770c3f4c58f7b4f6c.png with policy now +2023-07-12 00:41:01,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:01,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:01,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:01,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:02,113 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r15h5x44-media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 00:41:02,155 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wb8y7n3r-media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 00:41:02,164 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tlw86rco-media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 00:41:02,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:02,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:02,199 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xs3lziqw-media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 00:41:02,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:02,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 00:41:02,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 00:41:02,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 00:41:02,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 00:41:02,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:41:03,034 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3qlhbqq-media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 00:41:03,042 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yo57wtq9-media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 00:41:03,048 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hrwqwzb6-media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 00:41:03,105 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tll9zjp9-media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 00:41:05,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:06,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:08,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:41:08,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:10,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:11,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:11,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:12,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:16,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:17,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:17,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:18,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:20,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:21,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:21,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:21,306 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:21,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:21,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:22,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:22,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:24,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:26,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:27,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:30,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:31,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:31,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:31,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:31,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:31,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:32,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:32,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:32,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:32,892 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:34,731 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:36,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:38,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:38,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:41:40,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:41,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:41,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:41,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:41,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:41,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:42,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:43,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:44,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:47,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:41:47,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:41:48,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:48,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:50,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:51,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:41:51,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:41:51,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:41:51,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:41:51,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:41:52,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:53,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:41:54,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:56,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:41:58,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:00,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:01,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:01,466 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:01,466 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:01,468 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:01,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:02,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:03,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:03,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:04,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:04,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:08,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:42:08,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:09,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:10,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:12,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:12,031 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:12,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:12,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:12,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:12,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:14,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:16,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:18,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:18,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:18,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:19,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:20,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:21,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:21,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:21,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:21,981 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:22,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:22,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:24,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:26,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:28,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:30,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:30,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:32,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:32,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:32,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:32,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:32,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:33,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:33,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:34,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:35,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:36,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:37,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:42:37,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:42:37,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:42:37,534 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:42:37,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:42:38,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:42:38,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:40,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:40,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:42:45,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:49,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:42:49,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:42:51,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:42:56,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:01,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:04,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:04,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:06,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:08,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:43:11,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:16,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:19,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:19,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:22,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:27,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:31,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_897ca9ce40094218e736.png with policy now +2023-07-12 00:43:31,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_a6aefafa2d783d4803ef.png with policy now +2023-07-12 00:43:31,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_c063d5e9ec2283745c8b.png with policy now +2023-07-12 00:43:31,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:43:31,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_160_542fde61a9a7764e7963.png with policy now +2023-07-12 00:43:32,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:43:32,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:43:32,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:43:32,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:43:32,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:43:32,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 00:43:32,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 00:43:32,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 00:43:32,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 00:43:32,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:43:32,294 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a0lih3sw-media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 00:43:32,347 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5d0vwiv3-media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 00:43:32,402 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ecuuvjg-media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 00:43:32,449 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/drzfcr5i-media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 00:43:33,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:33,203 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mef4hn2q-media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 00:43:33,400 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3p30hdae-media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 00:43:33,693 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3nl0jkiu-media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 00:43:33,905 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jv0cgo68-media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 00:43:34,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:34,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:37,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:38,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:38,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:43:39,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:41,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:41,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:43:43,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:43,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:47,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:49,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:49,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:49,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:43:49,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:43:51,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:43:51,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:43:51,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:43:51,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:43:51,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:43:51,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:53,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:54,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:43:55,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:57,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:43:59,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:00,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:00,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:00,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:00,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:00,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:01,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:03,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:04,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:04,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:05,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:05,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:07,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:08,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:44:09,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:10,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:10,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:10,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:10,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:10,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:11,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:11,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:15,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:15,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:17,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:19,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:19,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:19,845 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:20,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:20,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:20,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:20,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:20,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:21,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:23,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:25,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:25,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:27,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:29,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:30,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:30,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:30,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:30,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:30,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:31,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:33,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:34,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:34,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:35,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:36,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:37,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:38,568 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:44:40,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:40,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:40,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:40,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:41,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:41,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:41,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:43,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:47,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:47,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:49,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:49,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:44:49,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:44:51,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:52,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:44:52,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:44:52,067 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:44:52,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:44:52,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:44:53,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:53,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:57,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:44:58,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:44:59,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:01,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:02,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:45:02,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:45:02,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:45:02,419 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:45:03,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:45:03,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:04,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:04,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:05,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:06,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:45:06,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:45:06,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:45:06,853 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:45:07,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:45:07,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:08,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:08,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:45:09,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:45:13,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:18,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:19,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:19,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:23,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:28,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:33,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:35,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:35,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:38,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:45:39,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:44,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:49,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:45:50,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:45:50,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:45:55,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:00,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:01,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_7e7561227c49152fdf02.png with policy now +2023-07-12 00:46:01,335 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,335 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_249e7863a11a9d2249ff.png with policy now +2023-07-12 00:46:01,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_582ff9cdb53e3daa2736.png with policy now +2023-07-12 00:46:01,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:46:01,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_170_80a5f2920f4985e1b760.png with policy now +2023-07-12 00:46:01,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:01,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:01,400 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:01,402 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:01,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:01,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 00:46:01,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 00:46:01,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 00:46:01,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 00:46:01,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:46:01,735 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ko8estw4-media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 00:46:02,044 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6v5nktoq-media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 00:46:02,154 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tgefjzy3-media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 00:46:02,273 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rafxeil0-media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 00:46:02,367 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r7o0qrel-media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 00:46:02,701 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nvwtn1vs-media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 00:46:02,716 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m7dz5om7-media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 00:46:03,068 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2adf7zp-media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 00:46:05,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:05,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:05,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:06,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:08,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:46:09,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:11,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:11,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:12,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:13,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:15,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:17,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:17,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:20,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:20,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:20,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:20,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:20,890 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:20,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:21,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:21,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:22,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:23,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:25,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:27,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:28,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:29,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:30,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:30,045 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:30,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:30,047 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:30,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:31,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:34,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:35,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:35,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:35,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:37,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:38,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:46:39,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:40,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:40,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:40,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:40,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:40,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:40,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:41,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:45,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:46,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:47,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:49,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:50,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:46:50,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:46:51,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:46:51,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:46:51,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:46:51,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:46:51,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:46:52,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:53,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:55,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:46:57,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:46:57,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:01,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:01,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:01,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:01,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:01,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:01,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:02,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:03,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:05,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:06,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:06,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:07,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:08,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:47:09,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:11,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:11,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:11,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:11,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:11,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:12,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:12,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:13,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:17,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:18,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:19,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:21,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:22,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:22,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:22,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:22,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:22,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:22,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:23,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:23,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:25,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:27,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:28,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:29,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:31,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:33,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:33,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:33,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:33,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:33,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:34,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:35,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:37,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:37,360 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:37,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:47:37,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:47:37,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:47:37,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:47:37,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:47:37,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:38,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:47:39,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:39,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:47:44,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:49,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:47:52,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:47:52,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:47:55,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:00,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:05,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:07,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:07,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:08,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:48:10,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:15,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:20,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:22,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:22,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:26,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:31,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:32,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_bcd830ba6ea2f7b921a8.png with policy now +2023-07-12 00:48:32,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,350 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_9f041d8a799e7d60ca1c.png with policy now +2023-07-12 00:48:32,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_2942fa312a4899f548b2.png with policy now +2023-07-12 00:48:32,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:48:32,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_180_57486bf611e806418a3a.png with policy now +2023-07-12 00:48:32,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:48:32,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:48:32,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:48:32,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:48:32,652 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/djabuexf-media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 00:48:32,665 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5qx4mf79-media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 00:48:32,684 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6bgxbbca-media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 00:48:32,705 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/acv6gxw6-media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 00:48:33,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:48:33,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 00:48:33,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 00:48:33,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 00:48:33,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 00:48:33,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:48:33,416 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qpoq2xxr-media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 00:48:33,425 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/84bbko12-media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 00:48:33,440 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/he91l0op-media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 00:48:33,440 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yg0qufqz-media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 00:48:37,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:37,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:37,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:38,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:38,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:48:40,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:42,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:42,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:48:42,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:44,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:46,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:47,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:48,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:51,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:48:51,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:48:51,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:48:51,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:48:52,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:48:52,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:52,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:48:52,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:48:53,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:48:54,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:56,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:58,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:48:58,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:01,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:01,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:01,776 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:01,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:02,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:02,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:03,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:04,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:07,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:07,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:07,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:08,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:49:09,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:09,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:11,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:13,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:13,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:13,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:13,026 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:13,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:15,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:15,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:17,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:19,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:20,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:21,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:22,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:22,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:23,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:23,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:23,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:23,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:24,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:25,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:25,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:27,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:29,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:30,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:31,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:32,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:32,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:32,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:32,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:33,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:33,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:36,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:37,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:37,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:37,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:38,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:49:39,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:41,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:41,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:42,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:42,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:42,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:42,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:43,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:43,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:47,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:47,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:49,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:52,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:49:52,879 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:49:52,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:53,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:53,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:49:53,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:49:53,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:49:53,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:49:54,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:49:55,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:57,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:49:58,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:49:59,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:03,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:04,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:50:04,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:04,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:50:04,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:50:04,043 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:50:04,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:50:05,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:08,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:08,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:08,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:50:09,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:50:09,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:09,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:50:09,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:50:09,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:50:09,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:50:09,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:11,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:50:14,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:19,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:23,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:23,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:25,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:30,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:35,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:38,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:50:38,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:38,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:40,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:45,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:50,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:50:53,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:50:53,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:50:56,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:01,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:03,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_1689c770ac115a929d63.png with policy now +2023-07-12 00:51:03,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,973 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_18d5cba2a1158df428aa.png with policy now +2023-07-12 00:51:03,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_c50d65d1788fc664335f.png with policy now +2023-07-12 00:51:03,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:51:03,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_190_2b5b9594c7937018b6be.png with policy now +2023-07-12 00:51:04,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:04,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:04,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:04,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:04,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:04,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 00:51:04,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 00:51:04,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 00:51:04,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 00:51:04,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:51:07,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:07,897 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a784hyj8-media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 00:51:08,118 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pafr7q4n-media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 00:51:08,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:51:08,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:08,895 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:09,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:10,301 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/21dbln40-media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 00:51:11,416 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qj2xzx8f-media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 00:51:11,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:12,419 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c99dzjed-media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 00:51:12,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:12,660 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v54t3t3r-media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 00:51:13,136 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xx1mc0ih-media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 00:51:13,156 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rmdavyr6-media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 00:51:13,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:13,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:15,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:17,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:18,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:21,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:23,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:23,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:23,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:23,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:23,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:23,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:23,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:23,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:24,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:25,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:27,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:29,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:29,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:31,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:33,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:33,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:33,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:33,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:33,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:35,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:35,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:37,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:38,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:51:38,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:38,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:39,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:40,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:43,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:43,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:43,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:43,829 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:43,831 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:44,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:45,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:46,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:47,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:49,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:52,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:53,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:51:53,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:51:53,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:51:53,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:51:53,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:51:53,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:53,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:51:53,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:51:55,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:57,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:51:57,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:51:59,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:03,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:03,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:04,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:04,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:04,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:04,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:04,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:05,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:07,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:08,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:52:08,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:09,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:09,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:11,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:13,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:14,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:14,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:14,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:14,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:14,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:14,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:15,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:17,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:19,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:21,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:23,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:24,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:24,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:24,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:24,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:24,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:24,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:24,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:24,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:25,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:27,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:29,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:31,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:33,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:34,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:34,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:34,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:34,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:34,816 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:34,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:35,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:38,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:52:39,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:39,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:39,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:52:39,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:52:39,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:52:39,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:52:39,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:52:39,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:39,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:41,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:52:45,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:50,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:52:54,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:52:54,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:52:56,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:01,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:06,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:08,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:53:09,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:09,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:11,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:16,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:21,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:24,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:24,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:27,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:32,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:33,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:33,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_50004396bfec8e96747d.png with policy now +2023-07-12 00:53:33,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:33,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_a321ae427f3731921032.png with policy now +2023-07-12 00:53:34,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:34,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_009b86ce19fa976b3f5d.png with policy now +2023-07-12 00:53:34,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:53:34,018 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_200_fe7b08ac2a5afbc11b18.png with policy now +2023-07-12 00:53:34,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:53:34,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:53:34,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:53:34,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_a321ae427f3731921032.png +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 00:53:34,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 00:53:34,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 00:53:34,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:53:34,275 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m3j85j9c-media/images/validation_200_a321ae427f3731921032.png +2023-07-12 00:53:34,282 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fg2w37d7-media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 00:53:34,336 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jipjvvr4-media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 00:53:34,348 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c95uy6ly-media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 00:53:34,642 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/826u4wm8-media/images/validation_200_a321ae427f3731921032.png +2023-07-12 00:53:34,693 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tmx3q6kx-media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 00:53:35,173 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1pgxccs-media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 00:53:35,214 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qfjfp81t-media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 00:53:37,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:38,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:53:39,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:39,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:40,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:42,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:42,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:44,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:45,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:53:48,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:48,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:50,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:52,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:53,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:53:54,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:54,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:53:54,404 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:53:54,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:53:54,407 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:53:54,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:53:54,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:53:55,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:53:56,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:58,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:53:58,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:02,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:04,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:04,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:04,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:04,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:04,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:04,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:05,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:06,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:08,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:08,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:54:09,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:09,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:09,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:12,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:14,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:15,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:15,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:15,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:15,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:15,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:16,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:18,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:20,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:20,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:22,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:24,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:24,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:24,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:24,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:24,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:24,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:25,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:26,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:26,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:28,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:30,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:32,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:34,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:34,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:34,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:34,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:34,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:34,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:36,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:37,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:38,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:38,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:54:39,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:39,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:42,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:42,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:44,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:44,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:44,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:44,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:44,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:45,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:46,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:48,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:50,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:52,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:53,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:54:54,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:54,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:54:54,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:54:55,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:54:55,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:54:55,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:54:55,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:54:56,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:54:58,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:54:59,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:00,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:02,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:04,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:04,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:05,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:55:05,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:55:05,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:55:05,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:55:05,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:55:08,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:08,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:55:09,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:55:09,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:55:09,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:55:09,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:55:09,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:09,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:10,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:55:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:12,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:55:15,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:20,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:24,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:24,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:25,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:30,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:35,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:38,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:55:39,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:39,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:41,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:46,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:51,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:55:54,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:55:54,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:55:56,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:01,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:04,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_1f72d7f053af939e5276.png with policy now +2023-07-12 00:56:04,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_ade80344c97e522275e6.png with policy now +2023-07-12 00:56:04,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_b7889716d369f8c47db2.png with policy now +2023-07-12 00:56:04,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:56:04,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_210_30fb41bcb1ce089cb7f8.png with policy now +2023-07-12 00:56:04,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:04,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:04,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:04,123 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:04,374 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cfc04n82-media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 00:56:04,391 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p23aesx3-media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 00:56:04,411 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/511nam4n-media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 00:56:04,418 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cqm5wftm-media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 00:56:04,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:04,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 00:56:04,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 00:56:04,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 00:56:04,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 00:56:04,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:56:05,001 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jmn7z23o-media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 00:56:05,010 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pn608moc-media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 00:56:05,041 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2boiplso-media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 00:56:05,049 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qgcjrdo7-media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 00:56:07,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:08,596 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:56:08,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:09,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:09,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:12,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:12,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:13,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:14,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:16,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:18,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:18,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:20,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:22,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:23,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:23,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:23,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:23,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:23,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:23,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:24,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:24,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:26,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:28,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:30,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:32,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:32,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:32,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:32,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:32,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:32,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:34,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:34,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:36,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:38,597 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:56:39,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:39,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:39,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:40,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:42,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:42,466 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:42,466 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:42,468 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:42,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:42,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:44,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:45,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:46,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:50,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:51,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:52,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:56:52,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:56:52,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:56:52,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:56:52,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:56:52,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:54,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:56:54,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:56:54,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:56:56,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:56:56,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:00,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:01,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:02,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:03,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:03,106 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:03,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:03,108 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:03,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:04,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:06,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:07,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:08,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:57:09,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:10,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:10,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:11,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:12,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:13,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:13,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:13,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:13,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:13,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:13,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:15,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:18,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:20,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:22,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:24,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:24,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:24,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:24,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:24,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:25,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:25,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:25,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:26,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:28,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:29,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:30,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:34,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:34,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:34,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:34,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:34,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:34,772 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:35,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:36,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:38,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:38,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:57:39,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:57:39,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:57:39,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:57:39,431 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:57:39,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:40,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:57:40,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:40,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:40,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:42,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:57:45,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:50,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:57:55,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:57:55,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:57:55,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:00,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:05,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:08,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:58:10,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:10,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:10,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:15,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:20,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:25,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:25,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:26,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:31,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:33,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_f581c132868ccb4db30b.png with policy now +2023-07-12 00:58:33,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_1b6340fa1d30a762feb0.png with policy now +2023-07-12 00:58:33,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_58119209c10f17b2d40d.png with policy now +2023-07-12 00:58:33,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 00:58:33,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_220_7e556c2aaa57f2ce8afe.png with policy now +2023-07-12 00:58:33,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:58:33,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:58:33,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:58:33,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:58:34,160 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wtrufn3n-media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 00:58:34,192 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c3rfipes-media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 00:58:34,214 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/otfrs8eb-media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 00:58:34,225 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ewqgymju-media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 00:58:34,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:58:34,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 00:58:34,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 00:58:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 00:58:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 00:58:34,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 00:58:34,605 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c7caswnk-media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 00:58:34,607 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ymyiv7h1-media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 00:58:34,663 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/233hh82s-media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 00:58:34,665 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3a7781aq-media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 00:58:36,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:38,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:38,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:58:40,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:40,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:42,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:42,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:44,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:44,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:58:46,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:47,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:48,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:50,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:52,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:52,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:58:54,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:58:54,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:58:54,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:58:54,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:58:54,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:58:55,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:58:55,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:58:56,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:58,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:58:58,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:00,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:02,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:03,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:03,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:03,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:03,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:03,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:04,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:06,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:08,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:08,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:59:09,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:10,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:11,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:11,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:12,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:14,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:14,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:14,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:14,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:14,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:15,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:16,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:18,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:19,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:22,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:24,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:24,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:24,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:24,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:24,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:24,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:25,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:26,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:26,283 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:26,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:28,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:30,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:31,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:34,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:34,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:34,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:34,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:34,963 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:35,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:36,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:36,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:38,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:38,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 00:59:42,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:42,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:42,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:42,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:44,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:46,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:46,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:46,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:46,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:46,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:48,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:48,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:50,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:52,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:53,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 00:59:54,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:56,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 00:59:56,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 00:59:56,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 00:59:56,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 00:59:56,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 00:59:58,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 00:59:58,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 00:59:58,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 00:59:58,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:00,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:02,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:04,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:04,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:05,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:00:05,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:00:05,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:00:05,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:00:06,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:00:06,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:08,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:00:09,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:10,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:00:10,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:00:10,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:00:10,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:00:10,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:00:10,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:12,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:00:13,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:13,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:00:15,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:20,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:25,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:28,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:28,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:00:30,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:35,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:38,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:00:40,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:43,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:43,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:00:46,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:51,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:56,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:00:58,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:00:58,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:01,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:04,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_a1e3c8f3087f5e3012a5.png with policy now +2023-07-12 01:01:04,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_16d4939d62b11379b933.png with policy now +2023-07-12 01:01:04,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_9dcda9052cdd403f3897.png with policy now +2023-07-12 01:01:04,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:01:04,677 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_230_d631c6d270c38f7631a9.png with policy now +2023-07-12 01:01:04,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:04,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:04,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:04,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:04,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:04,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 01:01:04,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 01:01:04,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 01:01:04,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 01:01:04,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:01:04,970 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5ke4ir5m-media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 01:01:04,976 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dn4s0e9a-media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 01:01:04,981 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l9w9k5cf-media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 01:01:04,988 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/un9yskt0-media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 01:01:05,317 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/getpikyf-media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 01:01:05,342 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c2t95zfu-media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 01:01:05,355 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vrslw0cl-media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 01:01:05,665 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p9sewsmo-media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 01:01:06,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:08,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:01:08,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:12,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:12,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:13,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:13,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:14,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:14,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:16,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:17,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:18,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:20,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:22,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:24,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:24,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:24,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:24,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:24,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:24,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:26,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:28,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:28,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:28,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:28,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:30,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:32,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:33,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:33,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:33,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:33,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:33,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:34,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:34,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:38,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:01:38,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:40,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:40,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:42,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:44,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:44,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:01:44,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:44,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:44,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:44,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:44,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:46,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:47,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:49,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:51,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:51,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:55,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:01:55,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:01:55,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:01:55,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:01:55,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:01:55,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:57,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:01:57,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:59,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:01:59,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:01:59,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:02,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:03,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:04,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:04,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:04,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:04,842 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:05,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:05,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:07,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:08,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:08,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:02:09,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:13,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:14,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:14,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:14,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:14,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:14,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:14,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:14,948 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:15,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:15,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:17,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:19,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:19,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:23,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:24,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:25,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:25,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:25,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:25,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:25,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:25,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:27,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:29,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:29,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:29,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:29,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:31,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:35,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:35,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:35,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:35,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:35,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:35,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:35,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:37,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:38,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:02:40,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:02:40,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:40,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:02:40,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:02:40,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:02:41,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:02:41,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:43,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:02:44,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:44,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:02:45,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:50,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:55,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:02:59,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:02:59,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:00,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:05,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:08,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:03:11,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:14,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:03:14,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:16,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:21,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:26,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:29,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:03:29,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:31,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:34,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,731 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_5cfbeaed31a9337bdbad.png with policy now +2023-07-12 01:03:34,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,739 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_18282123a377a69416e7.png with policy now +2023-07-12 01:03:34,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,750 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_98861fa9aa94e0bb9466.png with policy now +2023-07-12 01:03:34,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:03:34,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_240_607b3eebed55e6abe830.png with policy now +2023-07-12 01:03:34,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:03:34,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:03:34,821 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:03:34,823 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:03:35,038 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pyasz1hw-media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 01:03:35,054 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fu7wmo5h-media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 01:03:35,065 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9m3pmamy-media/images/validation_240_18282123a377a69416e7.png +2023-07-12 01:03:35,096 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/orvb2iuf-media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 01:03:35,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:03:35,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 01:03:35,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 01:03:35,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_18282123a377a69416e7.png +2023-07-12 01:03:35,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 01:03:35,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:03:35,761 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z9om3u0x-media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 01:03:35,782 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/swo10992-media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 01:03:35,823 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ak8jej10-media/images/validation_240_18282123a377a69416e7.png +2023-07-12 01:03:35,836 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qtgtsad0-media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 01:03:37,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:38,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:03:40,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:42,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:43,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:44,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:03:44,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:44,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:03:44,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:03:46,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:48,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:50,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:53,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:03:53,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:03:53,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:03:53,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:03:53,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:03:54,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:03:54,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:56,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:58,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:03:58,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:00,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:00,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:02,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:03,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:04,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:04,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:04,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:04,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:04,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:04,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:08,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:08,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:04:08,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:10,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:12,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:14,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:14,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:14,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:14,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:14,765 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:15,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:15,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:16,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:16,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:18,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:19,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:20,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:24,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:25,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:26,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:26,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:26,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:26,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:26,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:26,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:28,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:30,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:30,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:30,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:32,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:34,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:36,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:36,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:36,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:36,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:36,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:36,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:38,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:38,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:04:40,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:41,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:42,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:46,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:46,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:04:46,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:04:47,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:47,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:47,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:47,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:47,300 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:48,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:48,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:50,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:52,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:53,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:54,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:04:57,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:04:57,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:04:57,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:04:57,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:04:58,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:04:58,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:04:58,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:00,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:01,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:01,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:02,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:03,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:04,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:07,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:05:07,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:05:07,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:05:07,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:05:08,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:05:08,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:08,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:05:08,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:10,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:12,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:05:12,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:05:12,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:05:12,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:05:12,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:05:12,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:13,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:14,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:16,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:05:16,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:16,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:18,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:23,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:28,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:31,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:31,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:33,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:38,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:05:39,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:44,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:46,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:05:46,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:05:49,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:54,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:05:59,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:01,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:01,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:05,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:06,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_96a516914a407fcfc504.png with policy now +2023-07-12 01:06:06,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_7a35da35c250d174b078.png with policy now +2023-07-12 01:06:06,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_8833dd1eeb120dd86be3.png with policy now +2023-07-12 01:06:06,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:06:06,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_250_cc24f9252c2efc365a1d.png with policy now +2023-07-12 01:06:06,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:06,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:06,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:06,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:06,711 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvxptxya-media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 01:06:06,760 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4i1d0nhb-media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 01:06:06,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:06,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 01:06:06,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 01:06:06,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 01:06:06,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 01:06:06,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:06:06,771 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ez37az7u-media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 01:06:06,803 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fa8i2bl8-media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 01:06:07,189 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m789jqjd-media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 01:06:07,203 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p4u4rz2y-media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 01:06:07,211 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lcjsx893-media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 01:06:07,247 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bvsjxzav-media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 01:06:08,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:06:11,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:12,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:14,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:16,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:16,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:16,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:16,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:16,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:20,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:22,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:22,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:24,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:26,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:26,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:26,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:26,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:26,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:26,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:28,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:28,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:30,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:31,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:31,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:33,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:34,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:35,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:35,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:35,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:35,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:36,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:36,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:38,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:06:38,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:38,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:40,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:42,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:44,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:46,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:46,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:46,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:46,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:46,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:46,846 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:46,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:06:46,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:06:48,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:50,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:50,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:54,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:55,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:06:56,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:06:56,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:06:56,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:06:56,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:06:56,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:06:56,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:06:58,872 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:00,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:00,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:02,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:02,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:02,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:05,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:05,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:05,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:05,463 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:06,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:06,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:06,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:08,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:07:08,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:10,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:11,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:14,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:16,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:16,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:16,353 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:16,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:16,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:16,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:17,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:17,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:17,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:20,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:22,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:22,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:24,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:26,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:26,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:26,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:26,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:26,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:26,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:27,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:30,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:32,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:32,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:32,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:33,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:34,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:37,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:37,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:37,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:37,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:38,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:07:38,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:38,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:38,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:40,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:42,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:07:42,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:07:42,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:07:42,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:07:42,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:07:42,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:43,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:44,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:07:47,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:07:47,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:07:49,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:54,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:07:59,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:02,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:02,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:04,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:08,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:08:09,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:14,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:17,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:17,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:20,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:25,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:30,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:32,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:32,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:35,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:36,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_9d432cf0ad1538825247.png with policy now +2023-07-12 01:08:36,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_c5ef4fbb8722ad5da91d.png with policy now +2023-07-12 01:08:36,937 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,937 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_4f96826bcb36d428d2d7.png with policy now +2023-07-12 01:08:36,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:08:36,938 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_260_c2fb4e9073164765d619.png with policy now +2023-07-12 01:08:36,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:08:36,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:08:36,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:08:36,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:08:37,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:08:37,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 01:08:37,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 01:08:37,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 01:08:37,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 01:08:37,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:08:37,226 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6haw5iwl-media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 01:08:37,235 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9f635joy-media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 01:08:37,255 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a90jd428-media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 01:08:37,287 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/usnqizk7-media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 01:08:37,618 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m0lz9qb8-media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 01:08:37,943 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cyjqvasr-media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 01:08:38,087 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1olrcr29-media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 01:08:38,279 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1p09kbfm-media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 01:08:38,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:08:41,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:41,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:43,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:45,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:46,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:46,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:08:47,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:48,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:08:48,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:08:49,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:52,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:53,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:55,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:55,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:08:55,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:08:55,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:08:55,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:08:57,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:08:57,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:08:57,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:08:59,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:01,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:03,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:03,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:03,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:05,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:05,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:05,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:05,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:05,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:07,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:07,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:08,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:08,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:09:09,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:13,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:14,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:15,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:16,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:16,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:16,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:16,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:17,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:17,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:18,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:18,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:19,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:19,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:21,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:25,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:25,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:26,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:26,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:26,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:26,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:27,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:27,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:29,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:30,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:31,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:33,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:33,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:35,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:35,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:36,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:36,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:36,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:36,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:37,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:37,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:38,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:09:39,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:40,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:43,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:45,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:46,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:46,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:46,898 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:46,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:46,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:47,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:47,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:48,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:09:48,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:09:51,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:51,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:53,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:55,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:09:56,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:09:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:09:56,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:09:56,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:09:56,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:09:57,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:09:57,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:01,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:02,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:03,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:04,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:04,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:05,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:07,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:10:07,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:07,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:10:07,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:10:07,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:10:08,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:10:09,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:10:09,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:11,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:12,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:10:12,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:12,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:10:12,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:10:12,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:10:13,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:10:13,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:15,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:10:18,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:19,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:19,542 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:23,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:28,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:33,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:34,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:34,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:38,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:10:38,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:43,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:48,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:49,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:10:49,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:10:53,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:10:58,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:03,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:04,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:04,761 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:07,001 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,001 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_3fe25e6178ac8d39607e.png with policy now +2023-07-12 01:11:07,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_630c030e959854fa6506.png with policy now +2023-07-12 01:11:07,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_5145710341bd620359f3.png with policy now +2023-07-12 01:11:07,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:11:07,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_270_18e1dc0d33fcdedb1fc5.png with policy now +2023-07-12 01:11:07,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:07,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:07,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:07,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:07,323 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fsx9p9w-media/images/validation_270_630c030e959854fa6506.png +2023-07-12 01:11:07,337 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kco4vrwe-media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 01:11:07,340 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x9rtpzoz-media/images/validation_270_5145710341bd620359f3.png +2023-07-12 01:11:07,397 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d6t71fj9-media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 01:11:07,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:07,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 01:11:07,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_630c030e959854fa6506.png +2023-07-12 01:11:07,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 01:11:07,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_5145710341bd620359f3.png +2023-07-12 01:11:07,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:11:07,799 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3wj9f05j-media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 01:11:07,804 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ibcr6is-media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 01:11:07,810 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tbf4s9d8-media/images/validation_270_630c030e959854fa6506.png +2023-07-12 01:11:08,089 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v00suy47-media/images/validation_270_5145710341bd620359f3.png +2023-07-12 01:11:08,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:11:09,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:11,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:13,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:15,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:15,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:16,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:17,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:19,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:19,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:20,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:21,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:23,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:25,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:26,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:26,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:26,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:26,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:26,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:27,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:27,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:31,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:31,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:33,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:34,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:34,761 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:35,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:36,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:37,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:37,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:37,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:37,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:38,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:11:39,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:39,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:41,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:42,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:43,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:47,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:47,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:47,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:47,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:47,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:47,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:49,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:11:49,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:11:49,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:49,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:51,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:53,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:55,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:57,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:11:58,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:11:58,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:11:58,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:11:58,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:11:59,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:11:59,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:11:59,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:01,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:03,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:04,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:04,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:04,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:05,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:07,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:07,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:07,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:07,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:07,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:08,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:12:10,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:10,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:12,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:14,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:16,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:16,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:17,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:17,047 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:17,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:17,049 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:18,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:18,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:19,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:19,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:21,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:22,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:24,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:27,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:28,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:28,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:28,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:28,054 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:28,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:30,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:30,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:32,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:32,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:34,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:35,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:35,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:37,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:38,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:38,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:38,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:12:38,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:38,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:38,655 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:40,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:40,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:42,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:43,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:12:43,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:12:43,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:43,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:12:43,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:12:44,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:12:44,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:46,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:12:48,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:50,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:12:50,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:12:54,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:12:59,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:04,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:05,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:05,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:08,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:13:09,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:14,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:19,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:20,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:20,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:25,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:30,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:35,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:35,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:36,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:37,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_4a948e463af013031d1f.png with policy now +2023-07-12 01:13:37,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_ef3aba43d9e8e71c9284.png with policy now +2023-07-12 01:13:37,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,340 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_c25f63eee5ef9870fbd4.png with policy now +2023-07-12 01:13:37,351 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:13:37,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_280_c142b41b461357b2de28.png with policy now +2023-07-12 01:13:37,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:13:37,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:13:37,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:13:37,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:13:37,656 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iop0n8hy-media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 01:13:37,664 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ot513gul-media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 01:13:37,664 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/77zszpwi-media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 01:13:37,676 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i2rp4v7x-media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 01:13:38,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:13:38,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 01:13:38,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 01:13:38,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 01:13:38,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 01:13:38,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:13:38,541 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpwy4och-media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 01:13:38,570 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/znar4tgh-media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 01:13:38,576 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/392asd5d-media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 01:13:38,579 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tyeabbhy-media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 01:13:38,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:13:42,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:42,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:44,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:46,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:47,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:47,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:13:50,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:51,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:13:51,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:13:52,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:52,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:54,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:56,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:13:57,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:13:57,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:13:57,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:13:57,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:13:58,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:13:58,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:13:58,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:00,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:03,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:04,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:06,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:06,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:06,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:06,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:06,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:06,858 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:06,860 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:08,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:08,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:08,637 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:14:09,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:10,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:14,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:15,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:16,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:16,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:16,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:16,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:16,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:18,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:18,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:20,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:22,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:22,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:22,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:24,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:26,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:26,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:28,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:28,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:28,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:28,144 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:28,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:30,315 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:31,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:32,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:34,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:36,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:37,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:37,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:37,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:37,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:37,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:38,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:38,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:38,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:38,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:14:40,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:42,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:42,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:44,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:47,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:48,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:48,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:48,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:14:48,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:14:48,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:52,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:52,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:52,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:14:52,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:14:54,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:58,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:14:58,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:14:59,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:14:59,245 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:14:59,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:14:59,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:15:00,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:15:00,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:02,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:06,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:08,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:08,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:15:08,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:08,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:08,804 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:09,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:15:09,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:15:09,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:15:09,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:15:10,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:15:10,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:13,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:14,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:15:14,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:15:14,165 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:15:14,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:15:14,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:15:14,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:16,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:15:19,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:23,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:23,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:25,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:30,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:35,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:38,640 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:15:38,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:38,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:40,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:45,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:50,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:15:54,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:15:54,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:15:56,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:01,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:06,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:08,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,314 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_ff574f4d8f5ce6c5ff1a.png with policy now +2023-07-12 01:16:08,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_b84e927620cd6b0cb2a8.png with policy now +2023-07-12 01:16:08,359 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_1c16b044a10bd74e136a.png with policy now +2023-07-12 01:16:08,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:16:08,361 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_290_1b1094343a18b1720c55.png with policy now +2023-07-12 01:16:08,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:08,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:08,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:08,391 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:08,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:08,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 01:16:08,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 01:16:08,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 01:16:08,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 01:16:08,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:16:08,612 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/esuty0jl-media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 01:16:08,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:16:08,675 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p8cw5ln0-media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 01:16:08,682 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xw1jzmzo-media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 01:16:08,700 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qwn58tv2-media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 01:16:09,009 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/75gemzuu-media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 01:16:09,028 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kvg81yi5-media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 01:16:09,042 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/soi9uhji-media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 01:16:09,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:09,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:09,336 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ku87wxs8-media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 01:16:11,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:12,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:14,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:16,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:18,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:18,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:20,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:22,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:22,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:24,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:24,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:24,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:26,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:27,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:27,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:27,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:27,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:28,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:28,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:28,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:32,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:33,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:34,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:37,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:37,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:37,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:37,074 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:38,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:16:38,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:38,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:39,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:39,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:39,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:40,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:44,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:44,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:46,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:47,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:47,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:47,352 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:47,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:48,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:48,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:50,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:52,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:54,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:16:54,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:16:54,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:55,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:16:56,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:16:57,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:16:57,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:16:57,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:16:57,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:16:58,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:16:58,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:00,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:00,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:04,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:05,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:06,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:07,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:07,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:07,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:07,725 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:08,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:17:08,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:08,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:09,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:09,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:17:11,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:12,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:14,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:16,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:16,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:18,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:18,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:18,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:18,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:18,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:20,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:21,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:22,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:24,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:24,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:17:24,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:26,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:27,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:27,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:27,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:27,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:27,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:28,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:28,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:32,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:33,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:34,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:36,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:37,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:37,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:37,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:37,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:38,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:17:38,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:38,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:38,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:40,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:40,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:17:42,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:17:42,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:17:42,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:17:42,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:17:42,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:17:42,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:44,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:44,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:17:49,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:54,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:17:55,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:17:55,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:00,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:05,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:08,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:18:10,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:10,275 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:10,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:15,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:20,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:25,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:25,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:25,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:30,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:35,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:37,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_a2de729812e74d6ab884.png with policy now +2023-07-12 01:18:37,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,416 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_872691b49d402d6a737f.png with policy now +2023-07-12 01:18:37,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_1dc75f8c8767180b4449.png with policy now +2023-07-12 01:18:37,437 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:18:37,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_300_1a544bb059e5c537308c.png with policy now +2023-07-12 01:18:37,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:18:37,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:18:37,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:18:37,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:18:37,725 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6k3q4jnq-media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 01:18:37,734 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9awc3om1-media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 01:18:37,765 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b6origw0-media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 01:18:37,779 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5qr01v9t-media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 01:18:38,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:18:38,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:18:38,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 01:18:38,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 01:18:38,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 01:18:38,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 01:18:38,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:18:39,295 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fzpkqyiw-media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 01:18:39,301 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/txg2q3vm-media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 01:18:39,321 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c8lk94mh-media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 01:18:39,325 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k8jo3dzv-media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 01:18:40,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:40,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:40,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:42,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:44,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:46,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:46,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:47,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:18:48,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:52,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:52,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:54,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:55,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:18:55,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:18:56,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:18:57,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:18:57,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:18:57,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:18:57,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:18:57,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:18:58,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:18:58,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:02,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:02,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:05,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:07,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:07,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:07,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:07,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:07,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:07,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:08,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:19:09,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:09,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:10,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:10,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:13,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:13,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:15,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:17,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:18,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:18,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:18,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:18,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:18,444 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:19,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:21,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:23,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:24,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:26,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:26,095 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:27,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:29,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:29,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:29,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:29,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:29,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:29,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:31,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:31,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:33,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:34,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:37,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:38,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:19:39,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:39,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:39,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:39,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:39,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:39,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:41,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:41,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:41,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:41,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:44,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:45,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:47,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:49,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:49,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:49,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:49,502 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:49,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:19:50,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:51,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:19:51,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:53,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:55,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:19:56,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:19:56,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:19:57,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:59,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:19:59,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:19:59,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:19:59,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:19:59,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:20:01,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:20:01,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:01,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:05,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:07,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:08,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:20:09,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:09,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:20:09,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:20:09,938 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:20:09,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:20:11,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:20:11,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:11,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:11,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:20:12,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:14,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:20:14,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:20:14,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:20:14,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:20:15,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:20:15,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:17,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:20:18,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:23,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:27,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:27,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:20:29,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:34,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:38,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:20:39,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:42,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:42,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:20:45,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:50,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:55,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:20:57,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:20:57,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:00,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:05,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:08,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:21:08,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:08,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_619cd7b6e4fcfa89810a.png with policy now +2023-07-12 01:21:09,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:09,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_5a10ea9a10a25e57135c.png with policy now +2023-07-12 01:21:09,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:09,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_43c929f6445607c5f8d2.png with policy now +2023-07-12 01:21:09,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:21:09,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_310_01b83aa2726efc837f38.png with policy now +2023-07-12 01:21:09,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:09,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:09,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:09,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:09,321 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/957kycom-media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 01:21:09,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:09,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 01:21:09,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 01:21:09,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 01:21:09,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 01:21:09,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:21:09,343 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t5jf5mtv-media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 01:21:09,355 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8hiirjsc-media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 01:21:09,362 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k8qg21yt-media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 01:21:09,713 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p62scj7h-media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 01:21:09,821 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nuwxwt50-media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 01:21:09,833 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cxn8401v-media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 01:21:10,067 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wxp8d9ml-media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 01:21:11,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:12,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:12,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:15,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:16,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:17,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:19,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:19,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:21,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:21,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:25,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:27,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:27,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:27,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:27,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:28,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:28,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:28,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:28,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:29,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:29,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:31,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:32,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:33,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:35,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:37,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:38,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:21:39,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:39,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:39,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:39,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:39,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:39,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:41,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:42,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:42,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:43,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:43,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:45,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:48,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:49,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:49,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:49,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:49,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:49,422 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:21:51,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:21:51,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:53,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:53,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:57,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:21:57,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:21:57,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:59,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:21:59,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:21:59,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:21:59,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:21:59,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:21:59,603 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:01,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:01,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:04,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:05,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:07,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:08,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:22:09,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:09,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:10,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:10,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:10,029 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:10,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:11,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:11,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:12,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:12,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:14,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:15,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:17,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:20,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:20,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:20,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:20,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:20,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:21,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:21,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:23,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:25,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:25,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:27,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:27,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:27,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:30,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:30,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:30,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:30,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:31,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:31,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:31,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:33,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:35,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:37,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:37,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:38,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:22:39,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:40,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:40,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:40,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:40,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:41,567 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:41,567 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:42,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:42,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:42,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:45,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:22:45,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:22:45,026 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:22:45,029 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:22:45,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:22:45,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:47,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:22:48,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:53,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:22:57,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:22:57,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:22:58,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:03,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:08,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:23:09,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:12,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:12,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:14,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:19,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:24,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:28,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:28,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:30,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:35,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:38,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:23:39,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_73ad6d9a88337e318469.png with policy now +2023-07-12 01:23:39,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,110 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_a373a58713d055f4d0ec.png with policy now +2023-07-12 01:23:39,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,120 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_71caeb46293b077a4cd4.png with policy now +2023-07-12 01:23:39,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:23:39,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_320_1627810819d85c5e49c5.png with policy now +2023-07-12 01:23:39,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:23:39,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:23:39,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:23:39,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:23:39,403 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bemzo8j1-media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 01:23:39,419 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/495ncr5b-media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 01:23:39,494 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rqvtcvo7-media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 01:23:39,520 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0oo3049m-media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 01:23:39,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:23:39,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 01:23:39,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 01:23:39,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 01:23:39,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 01:23:39,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:23:40,060 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4ssif8i-media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 01:23:40,086 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3mdjdm9f-media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 01:23:40,089 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6602igfz-media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 01:23:40,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:40,227 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5va3an5w-media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 01:23:43,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:43,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:43,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:45,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:45,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:48,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:23:49,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:51,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:51,751 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:53,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:55,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:23:57,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:23:58,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:23:58,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:23:59,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:23:59,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:23:59,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:23:59,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:23:59,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:23:59,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:01,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:02,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:03,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:05,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:08,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:08,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:24:09,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:09,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:09,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:09,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:09,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:09,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:11,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:13,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:13,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:14,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:14,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:17,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:18,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:19,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:20,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:20,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:20,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:20,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:21,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:21,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:23,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:25,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:27,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:28,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:29,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:29,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:29,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:30,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:30,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:30,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:30,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:31,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:31,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:34,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:35,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:37,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:38,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:24:39,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:39,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:40,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:40,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:40,159 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:40,161 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:41,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:41,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:43,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:44,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:44,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:44,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:47,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:49,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:24:49,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:24:49,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:24:49,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:24:49,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:24:49,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:50,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:51,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:53,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:56,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:24:57,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:24:59,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:24:59,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:24:59,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:00,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:25:00,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:25:00,309 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:25:00,311 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:25:01,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:01,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:25:01,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:05,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:06,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:07,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:08,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:25:09,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:25:09,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:25:09,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:25:09,773 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:25:09,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:25:09,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:11,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:11,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:13,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:14,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:25:14,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:25:14,260 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:25:14,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:25:14,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:25:14,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:25:15,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:25:15,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:17,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:17,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:25:22,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:27,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:29,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:25:29,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:25:32,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:37,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:38,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:25:43,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:44,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:25:44,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:25:48,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:53,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:25:58,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:00,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:00,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:04,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:08,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:26:08,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_4559efd7cd62cbc0d7bd.png with policy now +2023-07-12 01:26:08,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_fc7d6afb7198af162914.png with policy now +2023-07-12 01:26:08,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_7fbcfe2f1f50d8f10ad6.png with policy now +2023-07-12 01:26:08,767 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:26:08,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_330_103af67d972f2199c1fc.png with policy now +2023-07-12 01:26:08,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:08,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:08,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:08,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:09,095 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/db50nohj-media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 01:26:09,095 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wv89vtps-media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 01:26:09,095 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cruhfswn-media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 01:26:09,114 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dkduydz0-media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 01:26:09,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:10,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:10,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 01:26:10,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 01:26:10,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 01:26:10,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 01:26:10,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:26:10,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u3kgmjrh-media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 01:26:10,480 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f9omobly-media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 01:26:10,484 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oiyjfwr2-media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 01:26:10,491 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blsgy0yr-media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 01:26:14,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:15,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:15,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:15,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:16,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:18,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:18,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:20,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:22,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:25,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:26,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:28,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:28,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:28,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:28,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:28,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:30,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:30,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:30,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:30,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:32,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:34,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:36,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:36,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:38,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:38,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:38,402 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:38,402 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:38,404 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:38,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:26:40,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:40,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:42,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:44,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:45,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:26:45,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:26:46,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:48,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:48,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:48,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:48,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:48,552 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:26:50,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:26:50,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:53,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:54,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:56,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:26:58,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:26:59,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:26:59,511 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:26:59,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:26:59,514 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:00,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:00,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:00,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:00,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:02,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:04,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:04,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:08,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:08,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:27:09,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:09,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:09,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:09,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:09,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:10,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:10,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:12,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:14,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:14,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:15,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:15,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:18,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:19,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:20,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:20,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:20,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:20,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:20,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:22,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:22,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:24,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:25,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:28,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:30,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:30,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:30,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:30,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:30,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:30,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:30,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:30,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:30,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:32,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:36,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:36,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:38,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:38,669 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:27:40,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:40,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:40,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:40,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:40,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:40,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:41,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:42,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:45,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:27:45,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:27:45,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:27:45,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:27:45,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:27:45,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:27:46,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:27:46,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:46,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:48,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:27:51,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:27:56,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:00,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:00,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:02,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:07,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:08,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:28:12,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:15,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:15,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:18,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:23,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:28,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:30,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:30,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:33,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:38,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:38,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:28:39,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:39,973 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_f1d20714dd011423e767.png with policy now +2023-07-12 01:28:40,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:40,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_dc0f97f353b7e7ab6dca.png with policy now +2023-07-12 01:28:40,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:40,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_457d6bff4c2a7ae5f48a.png with policy now +2023-07-12 01:28:40,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:28:40,015 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_340_2a1bc7c87c68196530cf.png with policy now +2023-07-12 01:28:40,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:28:40,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:28:40,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:28:40,047 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:28:40,320 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o44o1d2l-media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 01:28:40,328 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eepiqkhw-media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 01:28:40,339 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4nilvx1p-media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 01:28:40,480 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fw8i0sc-media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 01:28:40,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:28:40,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 01:28:40,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 01:28:40,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 01:28:40,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 01:28:40,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:28:40,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w5id7nt3-media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 01:28:40,871 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dsmsynoh-media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 01:28:40,878 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o3zlux50-media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 01:28:40,885 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zul3r25-media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 01:28:43,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:45,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:28:45,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:28:46,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:48,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:48,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:50,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:28:50,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:52,564 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:54,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:56,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:28:59,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:28:59,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:28:59,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:28:59,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:28:59,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:00,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:00,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:01,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:01,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:02,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:04,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:04,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:06,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:08,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:29:09,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:10,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:10,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:10,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:10,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:10,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:10,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:12,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:14,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:14,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:16,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:16,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:16,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:19,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:19,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:19,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:19,887 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:19,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:20,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:20,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:22,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:24,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:25,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:28,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:30,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:30,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:30,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:30,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:30,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:30,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:30,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:32,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:32,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:32,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:35,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:36,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:38,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:29:38,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:40,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:41,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:41,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:41,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:41,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:41,253 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:42,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:44,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:46,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:46,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:48,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:29:48,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:29:48,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:50,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:51,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:29:51,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:29:51,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:29:51,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:29:52,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:52,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:29:54,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:56,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:29:58,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:29:58,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:00,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:00,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:30:00,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:30:00,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:30:00,887 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:30:02,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:30:02,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:03,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:03,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:03,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:06,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:08,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:30:08,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:08,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:10,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:10,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:30:10,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:30:10,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:30:10,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:30:12,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:30:13,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:14,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:16,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:30:16,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:30:16,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:30:16,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:30:16,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:30:16,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:18,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:18,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:18,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:19,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:20,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:30:24,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:29,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:33,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:33,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:35,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:38,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:30:40,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:45,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:48,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:30:48,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:30:51,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:30:56,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:01,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:03,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:03,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:07,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:08,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:31:11,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,366 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_22aac1153c97ec2693c5.png with policy now +2023-07-12 01:31:11,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_e283ca771222ebaf7b13.png with policy now +2023-07-12 01:31:11,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_bb4784ba59026990690d.png with policy now +2023-07-12 01:31:11,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:31:11,415 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_350_1c59f118bd1aa7b351ef.png with policy now +2023-07-12 01:31:11,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:11,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:11,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:11,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:11,679 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7zrw2eml-media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 01:31:11,717 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/66pbk1r6-media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 01:31:11,723 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5m9wumby-media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 01:31:11,740 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2d71trfl-media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 01:31:12,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:12,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:12,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 01:31:12,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 01:31:12,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 01:31:12,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 01:31:12,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:31:13,274 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grcofz36-media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 01:31:13,307 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wppdap8y-media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 01:31:13,315 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5a0b03r7-media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 01:31:13,334 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x96hd2ey-media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 01:31:16,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:18,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:18,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:18,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:18,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:20,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:21,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:23,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:24,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:26,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:28,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:28,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:31,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:31,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:31,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:31,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:31,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:33,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:33,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:33,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:33,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:35,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:38,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:31:38,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:39,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:40,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:40,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:40,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:40,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:41,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:41,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:43,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:44,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:47,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:48,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:31:48,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:31:49,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:50,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:52,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:31:52,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:31:52,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:31:52,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:31:53,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:31:53,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:55,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:31:55,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:31:57,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:01,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:01,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:02,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:02,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:02,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:02,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:03,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:03,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:04,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:04,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:05,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:07,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:08,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:32:09,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:11,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:12,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:12,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:12,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:12,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:12,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:13,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:13,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:15,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:17,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:19,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:20,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:20,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:21,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:22,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:23,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:23,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:23,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:23,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:23,729 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:25,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:27,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:27,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:29,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:31,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:32,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:33,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:33,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:33,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:33,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:33,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:35,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:35,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:35,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:37,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:38,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:38,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:32:39,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:41,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:43,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:43,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:43,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:43,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:43,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:43,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:45,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:47,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:47,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:32:47,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:32:47,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:32:47,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:32:48,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:49,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:32:49,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:50,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:32:50,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:32:51,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:32:54,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:32:59,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:04,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:05,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:05,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:08,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:33:09,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:14,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:19,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:20,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:20,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:24,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:29,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:34,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:35,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:35,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:38,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:33:40,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:41,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_26e2ebd54ecac5225fde.png with policy now +2023-07-12 01:33:41,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_d5ce8065e09a898bbb73.png with policy now +2023-07-12 01:33:41,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_1483311cc0a22899fd84.png with policy now +2023-07-12 01:33:41,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:33:41,971 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_360_09d003a6833146016be6.png with policy now +2023-07-12 01:33:42,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:33:42,001 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:33:42,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:33:42,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:33:42,240 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/snbslg5u-media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 01:33:42,287 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bwsi3t00-media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 01:33:42,299 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nuprcx0o-media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 01:33:42,303 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/af8uz5p9-media/images/validation_360_09d003a6833146016be6.png +2023-07-12 01:33:43,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:33:43,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 01:33:43,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 01:33:43,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_09d003a6833146016be6.png +2023-07-12 01:33:43,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 01:33:43,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:33:43,722 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wakw6gde-media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 01:33:43,746 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/14jl6msr-media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 01:33:43,760 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ib0rjqz3-media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 01:33:43,761 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icvuteh7-media/images/validation_360_09d003a6833146016be6.png +2023-07-12 01:33:45,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:47,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:49,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:50,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:33:50,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:33:50,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:51,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:33:51,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:55,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:56,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:33:57,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:33:59,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:01,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:01,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:01,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:01,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:01,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:01,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:01,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:05,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:05,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:05,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:07,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:07,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:08,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:34:09,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:11,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:11,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:11,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:11,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:11,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:11,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:12,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:15,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:17,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:17,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:19,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:21,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:21,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:21,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:21,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:21,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:21,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:21,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:22,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:23,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:25,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:27,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:28,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:29,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:31,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:31,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:31,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:31,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:33,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:33,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:33,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:35,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:36,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:36,993 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:37,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:38,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:34:39,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:41,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:42,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:42,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:42,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:42,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:43,552 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:43,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:44,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:45,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:47,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:50,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:51,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:52,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:34:52,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:34:52,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:34:52,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:34:52,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:34:52,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:34:53,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:34:53,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:55,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:34:56,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:34:59,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:01,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:01,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:02,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:35:02,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:35:02,746 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:35:02,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:35:03,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:35:03,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:05,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:07,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:07,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:07,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:08,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:35:09,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:11,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:12,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:13,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:35:13,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:35:13,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:35:13,090 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:35:13,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:35:13,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:15,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:17,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:35:17,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:17,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:35:17,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:35:17,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:35:17,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:35:19,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:35:22,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:22,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:23,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:28,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:33,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:37,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:37,482 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:38,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:38,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:35:43,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:48,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:52,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:35:52,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:35:53,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:35:58,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:03,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:07,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:07,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:08,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:36:09,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:11,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_66cc1fe983e4ffdf1c5b.png with policy now +2023-07-12 01:36:11,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_fa56e2608c00ceaca35e.png with policy now +2023-07-12 01:36:11,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_0b641787504f5e185899.png with policy now +2023-07-12 01:36:11,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:36:11,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_370_8ad08c11ab8ad22829d8.png with policy now +2023-07-12 01:36:11,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:11,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:11,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:11,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:11,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:11,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 01:36:11,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_0b641787504f5e185899.png +2023-07-12 01:36:11,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 01:36:11,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 01:36:11,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:36:11,813 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xs0fncrv-media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 01:36:11,848 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fu4qcgq-media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 01:36:11,858 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fvv0azz6-media/images/validation_370_0b641787504f5e185899.png +2023-07-12 01:36:11,860 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qk761rqn-media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 01:36:12,261 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f3uovqzy-media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 01:36:12,328 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zewmjw77-media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 01:36:12,611 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/77vnoo77-media/images/validation_370_0b641787504f5e185899.png +2023-07-12 01:36:12,659 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6br1axz-media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 01:36:15,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:15,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:17,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:19,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:20,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:21,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:22,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:22,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:23,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:25,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:25,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:27,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:29,846 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:30,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:30,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:30,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:30,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:31,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:31,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:31,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:33,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:37,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:37,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:37,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:37,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:38,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:36:39,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:40,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:40,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:40,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:40,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:41,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:41,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:42,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:43,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:45,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:48,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:49,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:50,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:36:50,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:36:50,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:36:50,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:36:51,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:36:51,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:53,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:36:53,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:36:53,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:55,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:57,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:36:59,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:36:59,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:00,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:00,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:00,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:00,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:01,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:01,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:04,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:05,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:07,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:08,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:08,352 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:08,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:37:09,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:10,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:11,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:11,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:11,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:11,208 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:11,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:11,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:15,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:16,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:17,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:19,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:21,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:21,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:21,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:21,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:21,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:22,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:22,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:23,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:23,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:26,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:26,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:28,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:30,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:31,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:31,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:31,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:31,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:32,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:32,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:34,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:36,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:38,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:38,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:37:38,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:38,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:38,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:40,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:41,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:41,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:41,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:41,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:42,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:43,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:44,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:46,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:46,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:37:46,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:37:46,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:37:46,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:37:48,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:37:48,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:37:49,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:53,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:37:53,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:37:54,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:37:59,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:04,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:08,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:38:08,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:08,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:09,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:14,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:19,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:23,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:23,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:25,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:30,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:35,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:38,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:38:39,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:39,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:40,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_8126eeaacd234695183b.png with policy now +2023-07-12 01:38:40,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_e3c3849a148163ace395.png with policy now +2023-07-12 01:38:40,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_150783a4c31c17364129.png with policy now +2023-07-12 01:38:40,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:38:40,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_380_cf0b270e00fd03447bc3.png with policy now +2023-07-12 01:38:40,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:38:40,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:38:40,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:38:40,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:38:41,057 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m6h2umbp-media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 01:38:41,079 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8xkpwe30-media/images/validation_380_150783a4c31c17364129.png +2023-07-12 01:38:41,084 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r2omgi7z-media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 01:38:41,090 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/455l7zqm-media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 01:38:41,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:42,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:38:42,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_150783a4c31c17364129.png +2023-07-12 01:38:42,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 01:38:42,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 01:38:42,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 01:38:42,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:38:42,602 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xaas1l9s-media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 01:38:42,616 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0g354fm5-media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 01:38:42,618 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ua05gi9h-media/images/validation_380_150783a4c31c17364129.png +2023-07-12 01:38:42,619 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1wskaieg-media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 01:38:44,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:47,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:48,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:50,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:50,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:38:52,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:52,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:54,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:38:54,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:38:54,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:56,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:58,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:38:58,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:38:59,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:38:59,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:38:59,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:38:59,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:00,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:02,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:03,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:04,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:06,279 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:08,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:08,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:39:09,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:09,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:09,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:09,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:09,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:09,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:09,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:10,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:10,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:12,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:14,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:16,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:18,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:19,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:19,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:19,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:19,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:20,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:20,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:20,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:24,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:24,208 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:24,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:25,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:26,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:28,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:30,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:30,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:30,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:30,201 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:30,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:31,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:32,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:34,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:36,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:37,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:38,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:38,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:39:39,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:39,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:40,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:40,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:40,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:40,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:40,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:42,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:42,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:44,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:46,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:47,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:50,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:50,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:39:50,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:39:50,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:39:50,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:39:52,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:39:52,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:52,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:54,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:39:54,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:39:54,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:39:58,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:39:58,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:00,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:01,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:40:01,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:40:01,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:40:01,545 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:40:02,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:40:02,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:03,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:06,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:08,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:08,699 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:40:08,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:09,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:09,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:10,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:12,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:40:12,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:40:12,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:40:12,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:40:12,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:40:14,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:14,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:16,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:17,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:40:17,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:40:17,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:40:17,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:40:18,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:40:18,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:40:19,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:24,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:24,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:25,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:30,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:35,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:38,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:40:39,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:39,562 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:40,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:45,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:50,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:40:54,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:40:54,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:40:55,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:00,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:05,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:08,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:41:09,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:09,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:10,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:11,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_4f13ece0060ef305e7c1.png with policy now +2023-07-12 01:41:11,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_cece94bd69f475704bfa.png with policy now +2023-07-12 01:41:11,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_b80214c7e7fafa67c5b0.png with policy now +2023-07-12 01:41:11,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:41:11,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_390_dea73b102aef933ebfb7.png with policy now +2023-07-12 01:41:11,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:11,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:11,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:11,627 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:11,845 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oppjbn4g-media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 01:41:11,850 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4giqmlqj-media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 01:41:11,865 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q12qum9w-media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 01:41:11,886 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m8fo4d41-media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 01:41:12,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:12,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 01:41:12,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 01:41:12,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 01:41:12,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 01:41:12,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:41:12,993 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1ehuax9-media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 01:41:13,003 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qnspzhnt-media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 01:41:13,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/woxvzgwk-media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 01:41:13,019 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ry33suft-media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 01:41:16,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:16,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:18,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:20,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:21,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:21,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:22,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:24,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:24,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:26,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:26,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:28,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:30,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:31,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:31,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:31,219 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:31,221 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:32,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:32,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:32,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:34,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:37,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:38,704 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:41:38,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:40,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:40,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:40,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:40,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:40,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:40,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:40,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:42,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:42,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:44,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:48,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:48,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:50,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:41:50,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:41:50,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:41:50,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:41:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:41:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:52,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:53,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:41:54,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:56,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:41:56,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:41:58,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:41:58,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:00,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:02,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:02,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:02,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:02,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:02,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:04,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:04,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:06,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:08,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:42:09,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:11,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:11,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:11,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:13,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:13,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:13,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:13,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:13,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:15,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:15,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:15,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:19,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:20,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:21,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:23,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:23,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:23,704 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:23,704 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:23,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:25,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:25,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:25,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:26,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:26,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:29,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:30,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:31,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:33,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:33,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:33,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:33,943 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:33,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:35,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:35,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:36,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:38,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:42:39,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:41,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:41,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:41,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:41,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:43,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:43,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:43,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:43,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:43,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:45,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:45,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:47,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:47,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:48,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:42:48,192 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:42:48,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:42:48,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:42:49,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:42:51,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:42:52,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:42:56,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:42:56,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:42:58,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:03,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:08,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:43:09,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:11,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:11,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:14,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:19,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:24,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:26,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:26,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:29,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:34,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:38,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:43:40,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:41,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:41,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:41,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,862 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_b8bc388d0a6aa1ae82dc.png with policy now +2023-07-12 01:43:41,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_301c182c9a0503b9c552.png with policy now +2023-07-12 01:43:41,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_d5487564dfd56ce35ceb.png with policy now +2023-07-12 01:43:41,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:43:41,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_400_5cbb16c19b007f3c49d3.png with policy now +2023-07-12 01:43:41,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:43:41,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:43:41,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:43:41,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:43:42,154 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0oohqkpu-media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 01:43:42,221 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kzc53a4z-media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 01:43:42,223 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sqkbvcsi-media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 01:43:42,223 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x6k645gi-media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 01:43:43,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:43:43,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 01:43:43,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 01:43:43,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 01:43:43,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 01:43:43,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:43:43,811 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mdlvoeb8-media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 01:43:43,821 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rjor2gp0-media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 01:43:43,824 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/00950s3e-media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 01:43:43,835 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3zyxzb6c-media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 01:43:46,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:47,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:49,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:51,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:51,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:52,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:43:53,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:56,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:43:56,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:43:57,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:43:57,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:43:59,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:00,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:00,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:00,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:00,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:01,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:01,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:03,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:03,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:05,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:07,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:08,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:44:08,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:09,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:10,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:10,400 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:10,400 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:10,402 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:11,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:12,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:12,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:44:13,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:14,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:15,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:17,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:19,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:20,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:20,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:20,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:20,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:20,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:21,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:23,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:25,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:26,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:27,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:27,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:27,696 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:44:30,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:30,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:30,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:30,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:31,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:31,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:31,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:33,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:35,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:37,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:37,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:38,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:44:40,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:40,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:40,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:40,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:41,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:41,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:42,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:42,696 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:44:43,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:43,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:45,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:47,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:49,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:50,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:44:50,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:44:50,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:44:50,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:44:51,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:44:53,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:54,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:44:55,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:57,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:44:57,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:44:57,971 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:00,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:01,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:45:01,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:45:01,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:45:01,261 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:45:01,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:45:01,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:03,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:05,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:06,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:07,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:08,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:45:11,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:11,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:11,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:45:11,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:45:11,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:45:11,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:45:13,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:45:13,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:14,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:14,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:16,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:17,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:45:17,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:45:17,029 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:45:17,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:45:17,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:45:17,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:19,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:45:22,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:27,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:29,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:29,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:33,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:38,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:38,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:45:43,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:44,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:44,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:45:49,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:54,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:59,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:45:59,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:45:59,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:04,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:08,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:46:09,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:11,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_36160e0e12a477c07ef9.png with policy now +2023-07-12 01:46:11,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_975e274aed0125eac405.png with policy now +2023-07-12 01:46:11,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_c88421f1a33201c27e7f.png with policy now +2023-07-12 01:46:11,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:46:11,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_410_3032692fd49e0f4ffa6a.png with policy now +2023-07-12 01:46:11,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:11,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:11,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:11,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:11,486 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpzv8jh1-media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 01:46:11,519 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n8bjgfj9-media/images/validation_410_975e274aed0125eac405.png +2023-07-12 01:46:11,533 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ei2lrzv7-media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 01:46:11,538 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f67wemps-media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 01:46:11,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:11,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 01:46:11,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 01:46:11,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 01:46:11,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_975e274aed0125eac405.png +2023-07-12 01:46:11,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:46:12,189 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sw3un76j-media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 01:46:12,191 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vb4grlcg-media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 01:46:12,193 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xwwx3gln-media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 01:46:12,198 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fpw5shcu-media/images/validation_410_975e274aed0125eac405.png +2023-07-12 01:46:14,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:14,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:15,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:15,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:17,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:20,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:21,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:21,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:23,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:25,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:26,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:27,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:29,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:29,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:29,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:31,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:31,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:31,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:31,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:31,898 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:32,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:33,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:35,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:37,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:37,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:38,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:46:39,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:40,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:40,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:40,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:40,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:41,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:43,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:43,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:44,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:44,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:45,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:47,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:48,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:49,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:50,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:46:50,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:46:50,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:46:50,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:46:51,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:46:53,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:53,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:55,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:57,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:46:59,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:46:59,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:46:59,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:46:59,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:00,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:00,406 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:00,406 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:00,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:01,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:01,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:04,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:05,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:07,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:08,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:47:10,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:10,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:10,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:10,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:10,581 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:10,583 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:12,044 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:14,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:15,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:47:15,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:47:15,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:16,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:18,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:20,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:21,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:21,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:21,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:21,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:22,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:22,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:24,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:26,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:26,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:30,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:30,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:47:30,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:47:31,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:31,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:31,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:31,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:31,956 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:32,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:32,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:34,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:36,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:38,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:38,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:47:40,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:42,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:42,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:42,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:42,124 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:42,126 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:42,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:44,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:44,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:46,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:47:46,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:47:46,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:47:46,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:47:46,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:47:46,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:47:47,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:48,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:47:48,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:50,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:47:52,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:47:57,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:01,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:01,270 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:03,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:08,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:08,720 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:48:13,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:16,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:16,270 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:19,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:24,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:29,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:31,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:31,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:34,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:38,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:48:39,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:41,324 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_2a287e605f741cc4ba84.png with policy now +2023-07-12 01:48:41,333 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,333 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_f17cfd3caf204c385f2c.png with policy now +2023-07-12 01:48:41,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_b5f6b2004b8fac7ef053.png with policy now +2023-07-12 01:48:41,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:48:41,353 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_420_e80fbc111f2744e80e0e.png with policy now +2023-07-12 01:48:41,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:48:41,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:48:41,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:48:41,395 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:48:41,635 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hh5uxqrf-media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 01:48:41,656 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02zi0f83-media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 01:48:41,662 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/78gdotye-media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 01:48:41,762 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/894vbkvv-media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 01:48:42,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:48:42,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 01:48:42,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 01:48:42,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 01:48:42,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 01:48:42,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:48:42,654 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1p5s7w36-media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 01:48:42,656 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3l897u41-media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 01:48:42,692 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ihfrbij7-media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 01:48:42,990 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7j9nyjin-media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 01:48:45,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:46,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:46,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:48:46,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:48:50,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:51,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:51,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:48:52,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:54,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:56,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:48:57,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:48:58,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:00,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:00,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:00,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:00,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:00,918 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:01,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:01,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:02,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:02,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:04,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:06,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:08,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:08,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:08,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:49:11,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:11,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:11,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:11,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:12,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:12,402 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:13,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:14,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:16,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:16,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:16,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:18,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:18,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:21,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:21,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:21,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:21,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:22,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:22,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:23,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:24,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:26,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:28,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:29,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:31,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:31,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:31,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:31,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:31,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:31,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:32,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:32,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:34,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:34,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:36,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:38,725 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:49:39,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:40,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:42,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:42,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:42,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:42,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:42,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:42,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:44,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:45,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:46,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:49:46,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:49:46,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:50,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:50,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:52,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:49:52,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:49:52,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:49:52,229 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:49:52,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:49:52,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:54,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:49:55,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:49:58,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:00,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:01,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:01,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:01,875 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:02,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:50:02,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:50:02,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:50:02,480 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:50:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:50:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:04,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:06,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:08,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:50:08,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:10,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:11,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:13,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:50:13,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:50:13,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:50:13,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:50:14,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:50:14,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:16,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:16,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:16,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:16,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:18,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:50:18,669 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:50:18,669 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:50:18,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:50:18,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:50:18,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:20,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:22,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:22,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:50:27,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:31,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:31,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:32,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:37,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:38,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:50:43,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:46,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:50:46,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:50:48,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:53,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:50:58,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:01,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:01,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:03,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:08,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:51:09,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:12,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,947 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_582c2e8324c4a214eee2.png with policy now +2023-07-12 01:51:12,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 01:51:12,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_31a8bb8062237a4e5e03.png with policy now +2023-07-12 01:51:12,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 01:51:12,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_cc52ea02c073b4adb746.png with policy now +2023-07-12 01:51:12,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 01:51:12,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:51:12,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_430_466a9d653c954a92a2d2.png with policy now +2023-07-12 01:51:12,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 01:51:12,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:51:13,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:13,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:13,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:13,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:13,296 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bb0yy0bx-media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 01:51:13,297 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pr4bkhz5-media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 01:51:13,306 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sm1o58l7-media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 01:51:13,328 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cw1slxpm-media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 01:51:13,798 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/emyq9upx-media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 01:51:13,804 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jdgoarta-media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 01:51:14,138 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nawlxknc-media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 01:51:14,167 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7oyckd7-media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 01:51:14,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:15,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:16,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:16,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:19,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:20,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:21,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:22,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:23,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:25,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:25,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:29,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:30,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:31,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:31,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:31,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:32,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:32,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:32,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:32,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:35,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:36,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:37,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:38,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:51:41,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:42,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:42,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:42,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:42,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:42,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:43,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:43,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:45,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:46,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:51:46,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:51:47,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:47,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:52,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:51:52,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:52,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:51:52,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:51:52,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:51:53,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:51:53,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:55,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:51:58,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:51:59,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:01,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:01,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:01,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:02,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:02,245 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:02,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:02,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:03,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:03,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:03,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:05,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:07,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:08,731 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:52:08,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:11,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:13,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:13,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:13,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:13,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:13,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:14,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:15,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:16,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:16,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:17,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:19,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:19,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:21,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:24,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:24,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:24,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:24,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:25,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:25,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:25,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:27,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:29,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:30,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:31,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:31,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:31,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:33,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:33,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:33,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:33,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:35,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:35,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:36,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:37,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:38,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:52:39,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:41,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:41,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:43,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:43,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:43,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:43,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:45,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:45,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:46,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:52:46,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:52:47,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:47,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:48,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:52:48,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:52:48,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:52:48,523 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:52:49,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:52:49,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:51,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:52,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:52:53,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:52:57,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:02,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:02,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:03,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:08,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:08,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:53:13,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:17,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:17,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:19,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:24,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:29,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:32,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:32,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:34,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:38,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:53:39,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:42,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,642 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_996fc8410c2def6b1ad3.png with policy now +2023-07-12 01:53:42,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_b7c3984877608e60dcc5.png with policy now +2023-07-12 01:53:42,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_37feadf7c6d79ab08bb8.png with policy now +2023-07-12 01:53:42,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:53:42,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_440_850de8645d4e0ec1b327.png with policy now +2023-07-12 01:53:42,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:53:42,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:53:42,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:53:42,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:53:42,969 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lia4zt2x-media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 01:53:42,987 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uy0gmsja-media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 01:53:42,997 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vzqf9me4-media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 01:53:43,003 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/za5jvqzn-media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 01:53:43,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:53:43,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 01:53:43,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 01:53:43,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 01:53:43,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 01:53:43,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:53:43,805 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qduugorx-media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 01:53:43,808 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zqq92uns-media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 01:53:43,812 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0osuufyo-media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 01:53:43,909 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xubkmlkf-media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 01:53:45,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:47,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:47,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:53:47,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:53:49,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:51,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:51,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:53,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:53:55,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:56,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:53:57,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:53:59,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:01,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:02,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:02,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:02,402 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:02,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:02,405 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:02,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:02,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:03,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:03,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:05,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:07,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:07,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:08,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:54:11,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:11,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:11,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:11,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:11,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:13,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:13,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:13,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:15,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:17,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:17,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:17,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:19,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:21,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:22,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:22,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:22,256 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:22,258 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:23,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:23,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:25,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:25,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:29,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:30,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:31,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:32,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:32,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:32,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:32,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:32,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:32,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:33,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:33,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:35,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:37,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:38,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:54:39,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:40,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:43,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:43,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:43,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:43,638 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:43,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:43,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:45,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:46,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:47,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:48,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:54:48,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:54:49,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:51,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:52,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:53,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:54:53,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:54:53,677 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:54:53,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:54:53,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:54:55,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:57,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:54:57,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:54:59,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:01,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:02,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:03,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:03,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:03,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:55:03,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:55:03,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:55:03,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:55:03,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:55:05,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:07,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:08,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:08,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:55:09,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:13,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:13,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:14,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:55:14,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:55:14,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:55:14,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:55:15,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:55:15,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:17,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:18,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:18,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:18,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:19,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:55:19,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:55:19,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:55:19,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:55:19,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:55:21,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:55:23,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:28,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:33,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:33,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:34,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:38,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:55:39,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:44,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:48,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:55:48,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:55:49,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:55:59,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:03,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:03,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:04,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:08,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:56:10,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:13,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_62e3070332b5e7b50721.png with policy now +2023-07-12 01:56:13,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_129489423a36c578b83b.png with policy now +2023-07-12 01:56:13,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_0be657c2999bb3f2f0e8.png with policy now +2023-07-12 01:56:13,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:56:13,765 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_450_75c8cb1b49d83b55316d.png with policy now +2023-07-12 01:56:13,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:13,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:13,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:13,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:13,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:13,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 01:56:13,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 01:56:13,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 01:56:13,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_129489423a36c578b83b.png +2023-07-12 01:56:13,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:56:14,050 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o27mgdgc-media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 01:56:14,066 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xxklbi7r-media/images/validation_450_129489423a36c578b83b.png +2023-07-12 01:56:14,077 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yfo3n2um-media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 01:56:14,096 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sa699x8z-media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 01:56:14,429 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlk1ynf7-media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 01:56:14,458 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rfnhwf9b-media/images/validation_450_129489423a36c578b83b.png +2023-07-12 01:56:14,750 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p5z5f5d0-media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 01:56:14,783 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0of647xh-media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 01:56:15,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:18,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:18,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:19,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:21,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:21,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:24,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:25,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:26,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:27,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:30,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:32,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:32,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:34,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:34,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:34,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:34,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:34,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:34,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:34,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:36,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:36,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:38,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:38,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:38,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:56:42,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:43,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:43,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:43,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:43,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:43,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:44,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:44,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:46,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:48,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:49,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:49,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:56:49,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:56:50,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:54,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:54,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:56:54,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:56:54,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:56:54,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:56:54,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:56:56,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:56:56,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:56:58,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:00,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:00,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:04,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:04,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:04,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:04,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:07,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:07,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:07,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:07,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:08,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:08,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:57:10,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:12,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:13,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:14,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:14,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:14,729 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:14,730 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:14,731 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:16,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:16,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:18,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:19,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:19,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:20,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:22,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:23,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:24,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:24,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:24,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:24,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:24,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:26,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:26,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:28,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:29,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:32,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:34,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:34,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:34,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:34,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:35,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:35,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:35,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:35,196 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:36,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:36,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:38,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:57:39,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:40,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:42,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:44,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:44,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:44,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:44,717 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:44,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:45,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:46,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:46,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:49,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:57:49,564 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:57:49,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:57:49,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:57:49,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:57:49,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:57:50,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:57:50,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:51,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:57:52,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:57:56,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:01,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:04,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:04,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:06,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:08,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:58:11,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:16,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:19,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:19,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:22,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:27,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:32,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:34,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:34,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:37,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:38,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:58:42,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:44,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_a6c4a260b23627355952.png with policy now +2023-07-12 01:58:44,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_69bab95618e258fac08d.png with policy now +2023-07-12 01:58:44,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_de7722337cbd652fed8a.png with policy now +2023-07-12 01:58:44,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 01:58:44,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_460_387823557d6d5cb784bb.png with policy now +2023-07-12 01:58:44,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:58:44,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:58:44,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:58:44,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:58:44,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:58:44,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 01:58:44,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 01:58:44,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 01:58:44,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 01:58:44,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 01:58:44,581 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5twgfza8-media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 01:58:44,627 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i12k99im-media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 01:58:44,636 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l5wuygyo-media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 01:58:44,637 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tg2a2sql-media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 01:58:44,959 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iyznowyr-media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 01:58:45,079 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ddhl4rfv-media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 01:58:45,108 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kx6yjy59-media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 01:58:45,310 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n05eb11p-media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 01:58:48,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:48,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:49,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:58:49,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:58:50,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:53,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:58:53,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:58:54,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:56,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:58,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:58:59,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:00,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:02,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:03,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:03,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:03,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:03,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:04,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:04,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:05,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:05,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:05,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:08,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:08,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:59:10,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:11,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:14,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:14,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:14,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:14,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:14,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:14,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:16,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:16,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:18,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:20,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:21,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:21,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:21,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:24,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:24,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:24,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:24,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:24,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:26,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:26,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:26,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:28,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:31,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:32,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:34,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:34,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:34,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:34,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:34,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:36,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:37,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:37,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:38,748 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 01:59:40,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:42,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:42,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:44,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:45,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:45,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:45,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:45,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:46,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:46,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:47,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:48,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:52,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 01:59:52,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 01:59:52,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 01:59:52,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:54,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:55,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 01:59:55,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 01:59:55,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 01:59:55,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 01:59:56,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 01:59:56,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 01:59:57,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:00,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:02,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:03,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:04,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:05,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:00:05,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:00:05,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:00:05,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:00:06,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:00:07,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:07,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:08,751 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:00:08,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:08,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:10,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:12,780 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:14,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:14,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:16,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:00:16,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:00:16,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:00:16,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:00:16,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:00:18,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:20,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:20,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:00:20,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:00:20,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:00:20,736 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:00:20,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:00:20,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:22,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:22,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:22,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:00:25,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:30,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:35,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:37,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:37,588 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:38,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:00:40,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:45,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:50,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:00:52,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:00:52,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:00:56,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:01,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:06,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:07,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:07,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:08,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:01:11,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:14,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_f98b8202065d26c1ef2f.png with policy now +2023-07-12 02:01:14,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,805 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_ae66beb9ba6213202415.png with policy now +2023-07-12 02:01:14,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_729f9b8d68182ccc9b8c.png with policy now +2023-07-12 02:01:14,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:01:14,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_470_f3357ea3af4ad6b0deea.png with policy now +2023-07-12 02:01:14,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:14,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:14,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:14,844 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:15,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 02:01:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 02:01:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 02:01:15,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 02:01:15,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:01:15,106 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i8igoxlk-media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 02:01:15,125 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/knq2502k-media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 02:01:15,131 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1azt4lwt-media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 02:01:15,141 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ttz8z5c0-media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 02:01:15,456 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kk0ftgyr-media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 02:01:15,497 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ai8le3yf-media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 02:01:15,508 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fhqm9oz-media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 02:01:15,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k4fj52rc-media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 02:01:16,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:19,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:22,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:22,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:22,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:23,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:25,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:25,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:27,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:28,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:33,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:34,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:35,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:35,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:35,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:35,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:35,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:37,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:37,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:37,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:37,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:38,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:01:39,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:39,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:41,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:44,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:44,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:44,449 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:44,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:45,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:45,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:45,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:47,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:49,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:50,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:52,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:01:52,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:01:53,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:54,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:01:54,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:01:54,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:01:54,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:01:55,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:01:55,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:56,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:01:57,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:01:59,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:01,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:02,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:04,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:04,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:04,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:04,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:05,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:05,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:07,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:07,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:07,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:07,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:08,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:02:09,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:12,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:13,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:15,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:16,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:16,033 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:16,034 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:16,035 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:17,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:18,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:19,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:21,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:23,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:23,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:23,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:23,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:25,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:25,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:25,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:25,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:25,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:27,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:27,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:28,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:31,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:33,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:33,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:35,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:35,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:35,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:35,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:35,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:37,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:37,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:38,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:38,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:38,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:02:38,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:41,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:43,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:43,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:45,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:45,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:45,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:45,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:45,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:45,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:47,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:49,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:50,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:02:50,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:02:50,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:02:50,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:02:51,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:02:51,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:53,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:02:53,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:02:53,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:02:54,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:02:59,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:04,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:08,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:08,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:08,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:03:09,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:14,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:19,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:23,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:23,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:25,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:30,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:35,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:38,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:38,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:38,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:03:40,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:44,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_57793f0e6246b307afce.png with policy now +2023-07-12 02:03:44,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_36f12af806c3f1d50a42.png with policy now +2023-07-12 02:03:44,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_f389fd00788c87466ea2.png with policy now +2023-07-12 02:03:44,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:03:44,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_480_11e4b59458340be3e02c.png with policy now +2023-07-12 02:03:44,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:03:44,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:03:44,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:03:44,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:03:45,043 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/llqixg1w-media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 02:03:45,094 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jgesh98g-media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 02:03:45,119 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/94zh5aq4-media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 02:03:45,121 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uy9rc82m-media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 02:03:45,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:03:45,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 02:03:45,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 02:03:45,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 02:03:45,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 02:03:45,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:03:45,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:45,899 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c6vj5cms-media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 02:03:45,930 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2ueixcv0-media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 02:03:45,998 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l93wf6cj-media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 02:03:46,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2c1g795j-media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 02:03:49,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:51,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:51,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:53,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:03:53,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:03:54,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:03:55,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:56,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:03:57,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:03:59,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:01,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:02,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:03,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:03,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:03,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:03,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:03,736 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:05,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:05,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:08,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:08,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:08,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:08,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:04:09,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:11,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:13,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:14,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:14,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:14,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:14,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:14,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:15,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:17,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:19,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:19,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:21,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:23,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:23,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:23,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:24,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:24,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:24,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:24,189 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:25,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:25,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:27,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:29,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:30,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:31,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:34,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:34,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:34,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:34,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:35,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:35,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:35,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:37,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:38,759 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:04:39,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:39,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:39,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:41,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:43,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:45,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:45,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:45,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:45,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:45,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:45,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:46,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:47,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:49,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:52,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:53,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:54,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:04:54,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:04:55,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:04:55,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:04:55,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:04:55,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:04:55,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:04:55,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:57,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:04:57,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:04:59,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:03,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:03,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:05,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:06,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:05:06,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:05:06,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:05:06,407 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:05:07,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:05:08,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:05:08,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:09,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:09,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:09,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:11,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:13,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:13,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:15,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:16,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:05:16,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:05:16,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:05:16,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:05:17,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:05:19,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:19,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:21,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:05:21,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:05:21,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:05:21,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:05:21,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:05:21,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:23,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:24,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:24,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:24,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:25,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:05:29,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:34,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:38,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:05:39,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:39,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:40,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:45,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:50,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:05:54,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:05:54,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:05:55,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:00,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:05,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:08,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:06:09,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:09,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:15,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:16,028 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_e1df59c538a185bbc3ab.png with policy now +2023-07-12 02:06:16,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 02:06:16,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 02:06:16,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 02:06:16,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 02:06:16,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:06:16,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_bb00c284bf371c7cee01.png with policy now +2023-07-12 02:06:16,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_68abe5bccc0897c1b607.png with policy now +2023-07-12 02:06:16,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:06:16,074 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_490_47a8d03f0cc699178621.png with policy now +2023-07-12 02:06:16,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:16,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:16,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:16,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:16,337 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6zbwot3j-media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 02:06:16,371 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qsdmi0k8-media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 02:06:16,406 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bpwdpz3f-media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 02:06:16,448 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlymdhok-media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 02:06:16,824 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5n5omzl1-media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 02:06:16,837 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qamjx8ft-media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 02:06:16,859 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h2rs5wfa-media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 02:06:17,162 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yt8tlche-media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 02:06:18,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:21,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:21,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:24,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:24,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:24,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:24,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:27,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:27,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:30,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:32,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:33,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:33,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:33,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:33,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:33,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:36,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:36,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:38,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:38,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:06:39,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:40,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:40,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:42,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:44,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:44,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:44,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:44,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:45,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:45,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:49,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:51,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:54,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:06:55,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:06:55,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:06:55,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:06:55,306 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:06:55,308 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:06:55,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:06:55,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:06:57,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:06:57,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:00,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:00,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:03,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:06,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:06,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:06,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:06,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:06,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:06,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:08,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:07:09,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:09,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:10,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:10,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:12,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:12,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:15,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:16,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:16,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:16,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:16,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:17,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:18,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:21,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:22,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:24,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:26,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:26,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:27,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:27,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:27,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:27,192 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:27,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:27,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:28,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:30,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:33,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:33,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:37,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:37,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:37,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:37,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:38,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:38,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:07:39,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:39,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:41,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:41,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:42,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:44,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:45,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:47,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:47,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:47,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:47,465 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:48,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:48,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:49,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:51,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:52,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:07:52,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:07:52,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:07:52,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:07:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:07:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:07:54,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:07:56,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:07:56,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:07:57,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:00,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:05,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:08,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:08:10,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:11,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:11,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:16,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:21,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:26,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:26,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:26,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:31,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:36,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:38,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:08:41,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:41,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:42,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:46,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_03a01949302184444647.png with policy now +2023-07-12 02:08:46,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_0d74931567eda76fb139.png with policy now +2023-07-12 02:08:46,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,592 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_0d16f7c02c5cc655df6a.png with policy now +2023-07-12 02:08:46,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:08:46,603 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_500_89f5a8910a8c98de8cb9.png with policy now +2023-07-12 02:08:46,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:08:46,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:08:46,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:08:46,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:08:46,901 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nbtodcw8-media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 02:08:46,914 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gf0s2pf5-media/images/validation_500_03a01949302184444647.png +2023-07-12 02:08:46,927 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ozcl5cuy-media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 02:08:47,054 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/257d5qzc-media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 02:08:48,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:08:48,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 02:08:48,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_03a01949302184444647.png +2023-07-12 02:08:48,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 02:08:48,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 02:08:48,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:08:48,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:48,777 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n8tidecb-media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 02:08:49,002 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1clln2jp-media/images/validation_500_03a01949302184444647.png +2023-07-12 02:08:49,117 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6xxlzo5l-media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 02:08:49,136 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d40k6eq1-media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 02:08:51,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:54,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:08:54,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:56,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:08:56,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:08:56,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:08:57,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:08:59,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:00,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:03,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:05,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:06,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:06,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:06,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:06,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:06,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:06,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:08,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:09:09,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:11,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:11,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:11,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:12,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:15,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:16,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:16,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:16,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:16,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:16,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:18,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:18,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:21,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:22,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:24,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:26,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:26,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:26,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:26,588 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:26,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:26,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:27,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:27,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:27,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:30,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:33,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:33,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:36,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:36,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:36,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:36,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:36,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:36,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:38,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:09:38,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:39,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:41,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:41,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:42,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:44,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:45,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:47,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:47,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:47,551 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:47,553 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:48,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:48,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:49,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:51,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:54,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:09:54,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:09:56,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:09:56,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:09:57,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:09:57,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:09:57,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:09:57,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:09:57,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:09:57,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:00,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:00,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:03,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:06,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:06,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:07,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:10:07,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:10:07,249 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:10:07,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:10:08,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:10:09,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:10:09,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:11,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:12,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:12,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:12,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:15,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:17,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:18,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:10:18,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:10:18,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:10:18,043 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:10:18,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:10:18,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:21,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:22,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:22,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:10:22,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:10:22,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:10:22,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:10:24,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:10:24,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:27,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:27,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:27,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:10:28,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:33,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:38,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:38,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:10:42,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:42,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:43,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:48,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:53,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:10:57,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:10:57,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:10:59,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:04,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:08,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:11:09,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:12,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:12,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:15,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:16,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_3234686b4d18835995c9.png with policy now +2023-07-12 02:11:16,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_da96661dc21e252cae64.png with policy now +2023-07-12 02:11:16,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_34fac9bf4212cdd6d437.png with policy now +2023-07-12 02:11:16,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:11:16,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_510_e3e834df071a8745bc59.png with policy now +2023-07-12 02:11:16,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:16,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:16,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:16,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:17,245 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/850a2qea-media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 02:11:17,260 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgcp8qrj-media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 02:11:17,270 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kipxzq8l-media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 02:11:17,271 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4vy65n4f-media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 02:11:18,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:18,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 02:11:18,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 02:11:18,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 02:11:18,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 02:11:18,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:11:19,169 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kvfyyp72-media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 02:11:19,190 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/79czmzhx-media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 02:11:19,191 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zyi4gg64-media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 02:11:19,197 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bts2vy1u-media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 02:11:20,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:21,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:26,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:27,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:27,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:27,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:27,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:30,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:31,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:33,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:36,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:37,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:37,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:37,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:37,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:37,539 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:38,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:11:39,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:39,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:42,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:42,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:42,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:42,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:45,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:46,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:46,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:46,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:46,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:47,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:48,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:51,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:52,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:11:54,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:56,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:11:56,849 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:11:56,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:11:56,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:11:57,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:11:57,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:11:57,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:11:57,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:11:58,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:00,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:03,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:04,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:07,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:07,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:07,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:07,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:08,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:12:09,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:09,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:12,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:12,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:12,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:15,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:15,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:17,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:17,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:17,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:17,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:19,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:19,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:20,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:22,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:26,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:27,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:27,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:27,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:27,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:27,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:27,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:28,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:28,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:31,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:31,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:34,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:36,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:37,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:37,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:37,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:37,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:37,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:38,780 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:12:40,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:40,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:42,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:42,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:42,406 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:43,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:46,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:47,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:47,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:47,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:47,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:47,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:49,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:49,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:52,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:52,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:12:52,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:12:52,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:12:52,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:12:52,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:12:55,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:12:55,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:57,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:12:57,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:12:58,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:12:58,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:03,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:08,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:08,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:13:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:12,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:13,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:18,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:23,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:27,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:27,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:29,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:34,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:38,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:13:39,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:42,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:42,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:45,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:46,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_61f1eebc64efc60d8ae6.png with policy now +2023-07-12 02:13:46,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_8d7aa871f805bb466b02.png with policy now +2023-07-12 02:13:46,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_95ea6a7d25f4eeb9af52.png with policy now +2023-07-12 02:13:46,935 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:13:46,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_520_c82ca30fe9549b880c53.png with policy now +2023-07-12 02:13:46,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:13:46,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:13:46,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:13:46,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:13:47,214 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s3u0o8c4-media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 02:13:47,264 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o1ajwky8-media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 02:13:47,264 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bfq67dui-media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 02:13:47,268 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5otvxrun-media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 02:13:49,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:13:49,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 02:13:49,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 02:13:49,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 02:13:49,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 02:13:49,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:13:49,555 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lmepa3aa-media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 02:13:49,563 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xlalgt2q-media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 02:13:49,565 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u74qu6uz-media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 02:13:49,574 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kd1pdrbf-media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 02:13:50,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:52,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:13:55,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:13:56,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:13:57,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:13:57,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:13:57,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:13:58,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:01,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:02,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:04,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:06,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:06,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:06,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:06,677 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:07,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:07,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:07,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:08,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:14:10,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:12,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:13,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:16,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:17,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:17,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:17,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:17,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:18,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:19,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:22,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:23,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:25,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:26,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:26,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:26,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:26,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:27,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:28,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:28,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:29,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:34,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:34,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:37,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:37,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:37,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:37,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:37,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:37,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:38,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:14:39,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:40,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:42,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:42,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:43,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:45,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:46,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:47,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:47,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:47,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:47,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:14:49,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:14:49,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:50,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:52,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:55,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:56,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:14:58,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:14:58,075 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:14:58,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:14:58,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:14:58,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:14:58,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:14:58,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:01,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:01,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:01,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:04,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:07,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:07,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:08,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:15:09,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:15:09,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:15:09,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:15:09,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:10,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:10,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:12,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:13,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:13,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:15:16,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:18,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:18,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:15:18,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:15:18,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:15:18,959 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:19,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:19,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:22,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:23,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:15:23,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:15:23,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:23,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:15:23,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:15:25,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:15:25,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:28,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:15:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:28,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:15:33,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:38,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:15:39,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:43,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:15:44,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:49,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:54,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:15:58,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:15:58,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:00,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:05,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:08,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:16:11,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:13,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:13,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:16,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:17,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,942 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_d9608828cb52fcf1ad77.png with policy now +2023-07-12 02:16:17,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_a5b58d688f92b4fc1442.png with policy now +2023-07-12 02:16:17,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_001908c5ae0df54582c9.png with policy now +2023-07-12 02:16:17,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:16:17,987 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_530_9ebd350e55790857624d.png with policy now +2023-07-12 02:16:18,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:18,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:18,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:18,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:18,310 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jdpwucqz-media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 02:16:18,326 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0o3gz6pf-media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 02:16:18,326 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kf59cwr6-media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 02:16:18,328 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/drjzb9d8-media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 02:16:19,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:19,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 02:16:19,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 02:16:19,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 02:16:19,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 02:16:19,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:16:19,916 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qjz8pflj-media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 02:16:19,942 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/laichacm-media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 02:16:19,944 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d2yjwtr2-media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 02:16:19,964 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3xkefalr-media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 02:16:22,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:22,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:25,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:28,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:28,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:28,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:28,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:31,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:33,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:37,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:38,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:38,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:38,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:38,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:38,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:38,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:16:40,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:40,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:43,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:43,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:43,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:48,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:48,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:48,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:48,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:49,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:49,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:49,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:52,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:54,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:16:55,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:58,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:16:58,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:16:58,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:16:58,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:16:58,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:16:58,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:16:58,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:16:58,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:16:59,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:01,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:04,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:05,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:07,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:07,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:07,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:07,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:07,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:08,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:17:10,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:10,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:10,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:13,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:13,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:13,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:17:15,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:16,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:17,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:17,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:17,843 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:17,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:19,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:19,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:21,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:22,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:25,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:27,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:28,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:28,422 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:28,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:28,425 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:28,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:28,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:28,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:17:31,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:32,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:34,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:37,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:37,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:38,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:38,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:38,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:38,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:38,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:17:40,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:40,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:43,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:43,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:43,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:17:48,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:48,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:48,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:48,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:48,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:49,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:49,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:52,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:53,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:17:53,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:17:53,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:17:53,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:17:54,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:17:55,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:17:55,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:17:58,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:17:58,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:00,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:05,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:08,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:18:10,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:14,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:14,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:16,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:21,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:26,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:29,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:29,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:31,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:36,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:38,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:18:41,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:44,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:44,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:18:47,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:48,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_978f41b570f2d5fd48e5.png with policy now +2023-07-12 02:18:48,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_68ef6d897da63a408d01.png with policy now +2023-07-12 02:18:48,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_37a3b637808816ae2a1f.png with policy now +2023-07-12 02:18:48,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:18:48,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_540_fc4fe59f596766ac60b6.png with policy now +2023-07-12 02:18:48,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:18:48,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:18:48,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:18:48,206 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:18:48,437 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zb796zei-media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 02:18:48,441 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4yajszcc-media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 02:18:48,469 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5jpa7rt3-media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 02:18:48,561 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vl1xh8gx-media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 02:18:50,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:18:50,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 02:18:50,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 02:18:50,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 02:18:50,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 02:18:50,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:18:50,331 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l7pr2mte-media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 02:18:50,332 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xz55n6o2-media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 02:18:50,410 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ecq2pqb5-media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 02:18:51,232 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/onqdushy-media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 02:18:52,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:53,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:18:56,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:18:58,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:18:58,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:18:59,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:18:59,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:18:59,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:02,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:03,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:05,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:08,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:08,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:08,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:08,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:08,625 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:08,626 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:08,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:19:11,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:11,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:14,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:15,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:19:15,319 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:18,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:18,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:18,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:18,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:19,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:20,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:20,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:23,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:25,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:26,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:28,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:28,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:28,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:28,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:29,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:30,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:30,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:19:30,319 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:32,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:35,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:36,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:38,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:38,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:38,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:38,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:38,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:38,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:19:41,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:41,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:41,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:44,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:45,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:19:45,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:19:47,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:48,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:48,720 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:48,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:48,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:19:50,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:19:50,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:52,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:53,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:56,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:58,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:19:59,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:19:59,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:19:59,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:19:59,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:19:59,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:01,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:01,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:02,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:02,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:03,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:05,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:08,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:08,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:08,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:20:09,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:20:09,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:20:09,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:20:09,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:11,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:11,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:13,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:14,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:16,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:19,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:20,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:20,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:20:20,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:20:20,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:20:20,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:23,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:23,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:25,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:25,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:20:25,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:20:25,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:20:25,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:20:26,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:20:26,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:29,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:20:31,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:31,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:31,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:36,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:38,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:20:42,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:46,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:20:46,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:20:47,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:52,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:20:57,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:01,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:01,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:02,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:07,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:08,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:21:13,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:16,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:16,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:18,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:19,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_90a56ad8ada0c9c8274d.png with policy now +2023-07-12 02:21:19,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_116ac864f0f13e57d81b.png with policy now +2023-07-12 02:21:19,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_55055c73d3e5ab4513f3.png with policy now +2023-07-12 02:21:19,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:21:19,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_550_402084574a959b5e4266.png with policy now +2023-07-12 02:21:19,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:19,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:19,462 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:19,464 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:19,706 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mvsrk80n-media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 02:21:19,731 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ho5irbyn-media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 02:21:19,731 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vennmvx3-media/images/validation_550_402084574a959b5e4266.png +2023-07-12 02:21:19,745 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/88w99fuv-media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 02:21:20,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:20,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 02:21:20,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 02:21:20,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_402084574a959b5e4266.png +2023-07-12 02:21:20,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 02:21:20,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:21:20,835 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blolfqm1-media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 02:21:20,846 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wxwcrkyy-media/images/validation_550_402084574a959b5e4266.png +2023-07-12 02:21:20,847 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02pixcbi-media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 02:21:20,856 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mx66wllq-media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 02:21:24,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:26,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:29,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:29,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:30,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:32,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:32,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:32,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:35,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:35,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:38,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:38,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:38,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:38,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:38,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:38,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:38,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:21:40,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:41,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:44,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:46,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:47,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:47,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:47,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:47,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:47,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:47,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:21:47,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:21:50,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:50,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:51,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:53,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:56,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:21:57,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:21:57,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:21:57,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:21:57,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:21:57,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:21:59,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:21:59,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:02,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:02,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:02,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:02,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:05,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:07,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:07,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:07,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:07,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:07,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:08,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:08,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:08,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:22:12,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:14,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:17,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:17,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:17,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:17,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:17,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:17,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:17,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:17,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:20,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:20,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:23,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:23,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:26,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:28,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:28,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:28,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:28,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:28,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:29,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:32,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:32,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:32,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:33,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:35,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:38,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:38,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:38,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:38,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:38,625 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:38,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:38,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:38,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:22:41,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:43,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:44,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:48,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:22:48,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:22:49,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:49,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:49,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:49,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:49,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:50,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:50,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:53,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:54,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:22:54,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:22:54,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:22:54,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:22:54,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:22:56,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:22:56,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:22:59,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:03,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:03,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:05,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:08,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:23:10,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:15,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:18,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:18,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:20,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:25,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:30,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:34,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:34,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:36,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:38,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:23:41,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:46,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:48,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_607b86b7a89c36fdadfe.png with policy now +2023-07-12 02:23:48,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_a72665b3636d99570f6e.png with policy now +2023-07-12 02:23:48,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_a54da6547e3af36c2510.png with policy now +2023-07-12 02:23:48,759 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:23:48,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_560_d160b3bcabae5e649acf.png with policy now +2023-07-12 02:23:48,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:23:48,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:23:48,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:23:48,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:23:49,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:23:49,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:23:49,039 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpm5tpm3-media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 02:23:49,065 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/91ttg7pm-media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 02:23:49,066 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/04ig92hf-media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 02:23:49,074 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h3bs5q0t-media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 02:23:50,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:23:50,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 02:23:50,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 02:23:50,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 02:23:50,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 02:23:50,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:23:51,152 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hj2ahtyq-media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 02:23:51,186 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6tsnlkzn-media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 02:23:51,188 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l0whpmyp-media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 02:23:51,196 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hi2eseub-media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 02:23:51,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:53,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:23:56,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:23:57,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:23:58,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:02,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:02,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:04,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:04,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:05,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:07,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:08,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:08,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:08,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:08,765 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:08,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:24:08,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:08,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:11,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:12,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:14,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:17,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:17,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:19,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:19,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:19,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:19,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:19,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:19,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:20,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:20,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:23,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:23,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:26,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:28,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:28,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:28,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:28,938 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:28,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:30,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:30,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:33,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:34,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:34,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:34,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:38,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:24:39,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:39,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:40,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:40,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:40,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:40,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:42,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:45,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:45,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:48,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:49,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:24:49,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:24:51,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:51,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:51,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:24:51,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:24:51,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:24:51,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:24:54,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:24:54,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:24:56,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:24:57,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:01,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:01,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:01,400 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:01,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:01,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:03,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:03,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:04,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:04,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:06,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:06,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:08,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:25:09,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:11,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:11,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:11,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:11,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:11,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:12,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:12,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:15,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:17,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:19,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:19,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:21,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:22,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:22,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:22,104 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:22,106 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:23,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:24,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:24,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:26,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:25:26,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:25:26,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:25:26,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:25:27,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:25:27,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:28,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:30,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:25:33,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:34,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:34,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:38,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:25:38,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:43,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:48,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:49,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:25:49,220 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:25:54,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:25:59,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:04,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:04,220 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:05,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:08,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:26:10,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:15,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:19,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:19,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:21,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:21,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_6d9eeb6998ef5daf6dc1.png with policy now +2023-07-12 02:26:21,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,433 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_eb681e116684d158f76a.png with policy now +2023-07-12 02:26:21,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_e92644ed701dc0b0faa2.png with policy now +2023-07-12 02:26:21,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:26:21,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_570_9c89c3127ee7dbbf4a2a.png with policy now +2023-07-12 02:26:21,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:21,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:26:21,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:26:21,463 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:26:21,714 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x9sjy590-media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 02:26:21,746 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ojyff4ky-media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 02:26:21,762 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c6xyct0y-media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 02:26:21,773 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o53k0ost-media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 02:26:24,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:26:24,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 02:26:24,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 02:26:24,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 02:26:24,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 02:26:24,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:26:24,678 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4pyuyb9x-media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 02:26:24,679 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpxrbg1o-media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 02:26:24,716 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6a9ex3qp-media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 02:26:24,737 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ljdp5rk1-media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 02:26:26,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:27,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:30,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:31,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:32,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:33,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:34,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:34,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:37,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:38,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:26:39,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:42,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:42,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:42,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:26:42,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:26:42,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:26:43,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:45,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:26:45,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:48,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:49,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:26:49,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:49,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:26:51,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:52,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:26:52,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:26:52,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:26:52,153 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:26:54,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:26:54,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:26:54,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:26:57,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:00,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:00,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:02,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:02,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:02,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:02,729 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:03,402 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:03,402 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:04,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:04,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:06,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:06,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:08,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:27:09,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:11,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:12,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:12,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:12,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:12,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:12,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:15,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:15,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:16,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:18,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:19,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:19,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:21,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:21,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:23,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:23,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:23,080 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:23,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:24,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:26,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:27,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:30,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:31,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:33,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:33,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:33,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:33,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:33,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:33,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:34,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:34,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:36,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:37,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:38,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:27:39,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:42,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:42,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:42,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:42,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:42,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:42,855 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:45,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:45,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:48,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:48,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:49,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:27:49,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:27:51,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:53,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:53,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:27:53,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:53,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:53,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:54,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:27:54,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:57,564 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:27:58,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:27:58,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:27:58,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:27:58,379 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:27:58,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:00,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:28:00,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:28:03,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:28:03,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:04,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:04,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:09,325 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:28:09,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:14,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:19,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:20,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:20,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:25,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:30,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:35,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:35,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:35,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:38,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:28:40,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:45,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:50,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:28:50,273 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:28:51,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:52,400 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_b26c685674fa1336fbab.png with policy now +2023-07-12 02:28:52,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_fd8f16b41072ce3bceb4.png with policy now +2023-07-12 02:28:52,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_b60bdd2ea4c17924c6d0.png with policy now +2023-07-12 02:28:52,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:28:52,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_580_d75cf57bdd0e29b1f7a8.png with policy now +2023-07-12 02:28:52,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:28:52,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:28:52,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:28:52,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:28:52,727 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zfekdgeu-media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 02:28:52,758 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l6sui735-media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 02:28:52,773 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p4ss4kp6-media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 02:28:52,780 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wrqx9t6g-media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 02:28:54,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:28:54,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 02:28:54,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 02:28:54,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 02:28:54,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 02:28:54,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:28:55,068 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ayatjk8t-media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 02:28:55,076 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/thvaky0z-media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 02:28:55,092 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hzcbjc7l-media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 02:28:55,102 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m6rwhrrd-media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 02:28:56,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:28:57,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:00,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:02,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:02,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:03,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:05,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:05,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:06,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:07,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:08,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:29:09,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:12,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:12,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:12,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:12,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:12,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:12,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:13,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:15,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:18,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:20,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:20,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:21,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:21,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:21,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:21,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:21,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:21,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:23,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:24,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:27,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:29,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:31,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:31,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:31,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:31,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:33,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:33,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:35,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:35,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:35,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:36,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:38,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:29:39,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:40,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:41,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:41,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:41,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:41,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:42,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:42,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:45,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:45,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:48,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:50,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:29:50,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:29:50,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:51,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:29:51,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:29:51,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:29:51,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:29:51,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:29:51,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:54,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:29:55,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:29:57,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:00,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:01,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:01,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:01,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:01,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:01,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:03,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:03,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:05,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:05,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:06,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:06,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:08,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:30:11,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:11,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:11,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:11,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:11,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:12,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:12,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:15,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:17,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:18,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:20,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:20,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:21,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:22,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:22,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:22,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:22,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:22,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:24,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:24,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:27,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:30:27,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:30:27,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:30:27,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:30:27,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:27,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:30:27,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:30,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:30:32,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:35,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:35,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:39,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:30:39,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:44,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:50,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:30:50,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:30:54,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:30:59,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:04,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:05,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:05,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:08,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:31:10,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:15,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:20,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:20,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:20,985 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:21,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_a98fc70c70656fdfe63b.png with policy now +2023-07-12 02:31:21,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_08863b59853c2669a10c.png with policy now +2023-07-12 02:31:21,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_e05be95e710336b098d2.png with policy now +2023-07-12 02:31:21,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:31:21,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_590_06bde4208de6769e990b.png with policy now +2023-07-12 02:31:21,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:21,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:31:21,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:31:21,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:31:21,828 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6b4kczj4-media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 02:31:21,872 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gtdikyrq-media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 02:31:21,886 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/anaa6mct-media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 02:31:21,890 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2jroihs6-media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 02:31:22,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:31:22,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 02:31:22,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 02:31:22,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 02:31:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 02:31:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:31:22,539 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yt24yq0f-media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 02:31:22,540 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/og5is7th-media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 02:31:22,542 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hg9czu5g-media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 02:31:22,550 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kqprf96v-media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 02:31:25,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:26,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:28,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:31,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:31,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:31,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:34,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:36,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:36,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:37,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:37,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:38,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:31:40,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:41,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:41,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:31:41,447 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:31:41,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:31:42,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:43,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:31:43,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:46,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:47,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:49,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:51,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:31:51,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:31:51,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:31:51,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:31:51,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:31:51,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:31:52,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:31:53,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:31:55,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:58,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:31:58,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:01,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:01,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:01,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:01,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:01,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:04,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:04,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:04,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:06,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:06,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:07,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:08,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:32:09,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:10,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:12,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:12,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:12,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:12,090 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:13,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:13,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:15,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:19,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:20,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:21,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:21,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:22,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:22,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:22,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:22,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:22,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:25,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:25,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:26,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:28,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:31,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:31,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:32,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:32,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:32,717 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:32,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:34,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:34,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:36,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:37,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:37,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:37,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:38,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:32:40,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:41,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:43,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:43,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:43,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:43,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:43,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:43,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:47,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:49,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:52,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:32:52,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:32:52,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:52,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:52,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:52,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:52,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:52,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:55,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:55,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:32:57,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:32:57,324 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:32:57,324 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:32:57,326 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:32:57,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:32:58,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:32:58,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:33:01,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:33:02,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:07,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:07,312 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:08,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:08,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:33:13,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:18,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:22,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:22,312 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:24,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:29,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:34,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:37,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:37,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:38,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:33:39,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:44,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:49,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:51,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_e5844c86288b2606b645.png with policy now +2023-07-12 02:33:51,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_07de5acd372977760b4c.png with policy now +2023-07-12 02:33:51,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_bbb685abfd63c223246e.png with policy now +2023-07-12 02:33:51,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:33:51,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_600_6b5551cfcbdcbf8c81e5.png with policy now +2023-07-12 02:33:51,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:33:51,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:33:51,742 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:33:51,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:33:52,067 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ouec5xcw-media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 02:33:52,108 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sqemhzyd-media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 02:33:52,109 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fjld9v7q-media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 02:33:52,114 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kw8y1nmt-media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 02:33:52,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:33:52,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:33:52,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:33:52,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 02:33:52,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 02:33:52,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 02:33:52,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 02:33:52,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:33:52,938 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ugmg0lv-media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 02:33:52,974 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lajohbtq-media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 02:33:53,025 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jj6cgpk3-media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 02:33:53,025 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6lz8nz57-media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 02:33:55,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:33:55,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:33:58,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:00,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:01,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:01,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:04,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:06,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:07,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:08,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:08,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:08,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:34:11,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:11,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:11,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:11,560 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:11,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:13,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:13,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:16,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:19,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:21,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:21,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:21,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:21,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:22,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:22,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:23,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:23,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:25,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:28,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:28,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:31,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:31,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:31,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:31,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:31,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:31,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:33,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:34,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:37,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:38,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:34:38,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:39,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:39,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:42,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:42,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:42,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:42,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:43,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:43,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:44,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:46,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:49,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:49,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:52,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:34:52,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:34:52,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:34:52,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:34:52,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:34:52,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:54,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:34:54,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:34:54,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:34:55,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:58,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:34:59,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:01,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:02,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:02,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:02,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:02,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:04,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:04,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:05,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:07,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:08,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:35:09,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:09,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:10,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:10,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:12,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:12,406 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:12,406 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:12,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:13,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:13,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:15,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:16,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:19,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:21,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:22,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:22,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:22,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:22,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:22,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:22,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:24,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:24,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:25,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:26,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:27,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:35:27,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:35:27,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:35:27,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:35:28,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:35:31,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:35:31,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:36,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:38,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:35:39,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:39,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:42,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:47,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:52,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:35:54,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:35:54,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:35:58,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:03,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:08,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:08,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:36:09,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:09,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:14,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:19,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:21,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_1d981ecc323c30e20b81.png with policy now +2023-07-12 02:36:21,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_b9cc12355deddaad9779.png with policy now +2023-07-12 02:36:21,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_134e6d2a749fcacd43b7.png with policy now +2023-07-12 02:36:21,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:36:21,694 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_610_92f0f0898c20a134b3c3.png with policy now +2023-07-12 02:36:21,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:21,725 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:36:21,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:36:21,727 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:36:21,939 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2mwrbwlo-media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 02:36:21,996 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wf5lgkyb-media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 02:36:22,017 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83o8qkul-media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 02:36:22,018 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3z8fh6at-media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 02:36:23,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:36:23,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 02:36:23,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 02:36:23,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 02:36:23,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 02:36:23,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:36:23,388 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u72w6ne9-media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 02:36:23,400 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l92cm2h0-media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 02:36:23,405 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3t1d354-media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 02:36:23,422 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b30f4ze6-media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 02:36:24,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:24,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:25,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:26,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:29,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:31,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:31,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:32,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:35,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:37,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:38,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:38,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:36:39,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:39,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:41,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:41,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:36:41,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:36:41,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:36:41,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:36:41,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:43,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:44,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:47,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:48,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:50,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:50,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:36:50,915 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:36:50,915 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:36:50,917 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:36:53,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:36:53,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:54,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:36:55,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:36:55,143 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:36:56,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:59,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:36:59,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:01,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:01,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:01,067 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:01,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:02,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:02,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:04,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:08,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:37:10,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:10,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:10,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:11,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:11,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:11,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:11,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:11,312 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:14,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:14,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:15,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:17,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:20,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:21,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:21,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:21,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:21,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:21,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:23,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:26,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:26,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:26,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:26,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:29,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:32,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:32,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:32,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:32,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:32,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:32,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:35,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:35,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:37,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:38,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:38,836 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:37:41,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:41,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:41,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:42,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:42,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:42,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:42,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:42,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:44,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:44,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:47,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:50,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:53,335 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:53,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:54,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:54,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:54,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:54,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:56,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:56,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:37:56,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:37:56,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:37:58,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:37:58,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:37:58,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:37:58,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:37:58,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:37:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:37:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:38:02,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:38:04,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:08,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:38:09,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:11,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:11,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:15,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:20,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:25,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:26,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:26,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:31,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:36,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:38,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:38:41,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:41,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:42,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:47,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:52,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:53,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_77093872b2a62f3184c0.png with policy now +2023-07-12 02:38:53,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_f72e5adad3a7e2192d4a.png with policy now +2023-07-12 02:38:53,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_ed7326150a0c47d8bd9c.png with policy now +2023-07-12 02:38:53,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:38:53,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_620_ad1a36db317705c6c410.png with policy now +2023-07-12 02:38:53,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:38:53,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:38:53,248 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:38:53,250 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:38:53,483 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o2s5o74y-media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 02:38:53,510 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/et9ww5li-media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 02:38:53,521 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n7rki5jp-media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 02:38:53,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:38:53,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 02:38:53,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 02:38:53,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 02:38:53,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 02:38:53,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:38:53,545 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z1yfbp4j-media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 02:38:53,875 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qgakst8k-media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 02:38:53,888 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9adx56pr-media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 02:38:53,891 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8zwxp8cj-media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 02:38:53,917 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wu8qdtdl-media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 02:38:56,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:38:56,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:38:58,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:38:59,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:02,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:03,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:04,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:05,582 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:08,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:08,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:39:09,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:11,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:11,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:11,598 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:13,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:13,278 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:13,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:13,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:14,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:14,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:15,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:17,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:20,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:21,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:22,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:22,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:22,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:22,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:23,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:23,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:26,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:26,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:26,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:26,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:32,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:32,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:32,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:32,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:32,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:32,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:35,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:35,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:37,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:38,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:38,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:39:41,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:42,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:42,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:42,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:42,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:42,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:42,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:42,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:44,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:44,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:47,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:48,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:50,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:53,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:39:53,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:53,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:39:53,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:39:53,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:39:53,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:39:53,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:56,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:39:57,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:39:57,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:39:59,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:39:59,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:02,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:03,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:03,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:03,877 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:03,879 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:04,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:05,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:08,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:08,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:40:10,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:11,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:12,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:12,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:14,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:14,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:14,094 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:14,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:14,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:14,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:16,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:17,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:20,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:21,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:23,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:23,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:23,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:23,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:23,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:23,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:26,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:26,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:27,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:27,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:28,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:40:28,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:40:28,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:40:28,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:40:29,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:40:29,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:32,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:32,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:40:37,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:38,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:40:42,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:42,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:43,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:48,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:53,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:40:57,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:40:57,229 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:40:58,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:03,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:08,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:08,844 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:41:12,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:12,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:41:13,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:18,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:22,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,885 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_4f603f3934c8321cc8fb.png with policy now +2023-07-12 02:41:22,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_79440b6bdd999b2e130a.png with policy now +2023-07-12 02:41:22,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_da9c8b5a7202534ddcc5.png with policy now +2023-07-12 02:41:22,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:41:22,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_630_bdab7dddcf554aad01f9.png with policy now +2023-07-12 02:41:22,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:22,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:41:22,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:41:22,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:41:23,198 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ayf2sld-media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 02:41:23,232 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ow023sp8-media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 02:41:23,246 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eeu1g7m5-media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 02:41:23,251 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ol8k73wz-media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 02:41:23,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:24,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:41:24,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 02:41:24,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 02:41:24,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 02:41:24,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 02:41:24,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:41:24,355 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/73z87ue7-media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 02:41:24,358 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyii1t3u-media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 02:41:24,387 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jldt6cx3-media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 02:41:24,407 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/21kfly09-media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 02:41:27,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:27,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:27,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:41:29,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:30,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:32,638 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:33,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:34,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:36,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:38,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:41:39,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:40,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:41,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:41,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:41:41,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:41:41,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:41:42,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:41:42,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:42,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:42,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:41:45,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:45,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:48,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:51,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:51,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:51,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:41:51,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:41:51,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:41:51,916 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:41:54,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:41:54,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:56,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:41:57,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:41:57,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:41:57,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:00,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:01,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:01,383 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:01,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:01,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:02,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:03,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:03,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:06,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:08,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:08,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:42:09,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:12,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:12,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:12,333 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:12,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:12,335 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:12,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:12,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:13,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:15,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:15,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:18,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:18,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:23,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:23,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:23,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:23,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:24,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:24,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:24,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:28,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:28,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:29,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:30,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:33,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:33,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:33,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:33,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:33,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:34,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:36,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:36,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:38,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:42:39,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:39,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:42,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:43,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:43,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:44,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:44,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:44,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:44,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:45,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:45,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:48,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:51,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:51,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:54,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:54,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:54,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:54,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:42:54,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:42:54,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:56,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:42:57,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:42:58,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:42:58,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:42:59,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:42:59,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:42:59,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:42:59,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:43:00,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:43:00,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:43:01,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:06,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:08,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:43:12,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:13,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:13,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:43:18,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:23,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:28,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:28,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:28,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:43:33,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:38,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:43:39,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:43,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:43,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:43:44,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:49,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:53,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,756 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_49779e88089791939e26.png with policy now +2023-07-12 02:43:53,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_b64ccfa16d3aa44a2151.png with policy now +2023-07-12 02:43:53,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_56616402a26a74e9ca6a.png with policy now +2023-07-12 02:43:53,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:43:53,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_640_c1d1fb2c049800d964ec.png with policy now +2023-07-12 02:43:53,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:43:53,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:43:53,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:43:53,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:43:54,089 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7rt9zrt3-media/images/validation_640_49779e88089791939e26.png +2023-07-12 02:43:54,127 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lx1270oc-media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 02:43:54,127 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7eyl13ks-media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 02:43:54,132 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p66i4dn5-media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 02:43:54,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:43:54,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 02:43:54,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_49779e88089791939e26.png +2023-07-12 02:43:54,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 02:43:54,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 02:43:54,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:43:54,893 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mi2ekolj-media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 02:43:54,904 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/63ez2e38-media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 02:43:54,910 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/do3r4q1q-media/images/validation_640_49779e88089791939e26.png +2023-07-12 02:43:54,921 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uu4cw65r-media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 02:43:55,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:43:58,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:43:58,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:00,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:01,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:02,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:03,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:06,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:07,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:08,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:44:09,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:11,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:11,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:11,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:11,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:12,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:12,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:12,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:13,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:13,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:17,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:18,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:21,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:22,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:22,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:22,063 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:22,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:23,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:24,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:24,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:27,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:28,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:28,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:30,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:33,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:33,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:33,073 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:33,075 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:33,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:33,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:34,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:36,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:38,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:44:39,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:42,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:43,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:43,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:43,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:43,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:43,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:43,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:44:44,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:45,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:45,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:48,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:50,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:51,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:53,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:44:53,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:44:53,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:44:53,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:44:54,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:44:54,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:44:55,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:44:59,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:44:59,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:00,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:01,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:03,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:03,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:03,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:03,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:03,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:06,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:06,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:06,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:08,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:45:09,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:11,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:12,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:14,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:14,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:14,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:14,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:14,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:14,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:15,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:15,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:16,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:18,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:21,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:22,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:24,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:24,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:24,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:24,200 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:24,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:24,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:27,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:28,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:29,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:29,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:29,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:45:29,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:45:29,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:45:29,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:45:30,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:45:30,782 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:45:33,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:38,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:45:39,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:44,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:44,303 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:45:45,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:50,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:55,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:45:59,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:45:59,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:01,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:06,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:08,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:46:11,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:14,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:46:14,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:17,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:22,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:23,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_7416b90b79fecff5e4ca.png with policy now +2023-07-12 02:46:23,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_606f3fe637934aa6a15b.png with policy now +2023-07-12 02:46:23,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_2b23cf43dc0d03af66c6.png with policy now +2023-07-12 02:46:23,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:46:23,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_650_774f3994c62fdf69ce65.png with policy now +2023-07-12 02:46:23,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:23,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:46:23,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:46:23,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:46:23,801 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bj28yw0b-media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 02:46:23,806 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h4gop8cg-media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 02:46:23,808 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gjor9qhd-media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 02:46:23,838 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/88xjqo48-media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 02:46:24,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:46:24,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 02:46:24,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 02:46:24,940 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 02:46:24,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 02:46:24,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:46:25,263 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/70mh8c9e-media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 02:46:25,277 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhe5luxj-media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 02:46:25,286 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/013kphde-media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 02:46:25,309 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dl90gqyz-media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 02:46:27,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:29,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:46:29,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:30,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:33,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:33,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:34,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:36,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:38,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:38,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:46:39,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:42,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:43,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:43,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:46:43,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:46:43,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:46:44,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:45,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:46:45,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:46:45,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:46:45,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:48,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:49,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:52,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:53,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:46:53,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:46:53,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:46:53,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:46:55,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:46:55,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:46:55,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:46:58,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:00,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:00,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:00,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:01,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:03,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:03,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:03,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:03,131 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:04,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:04,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:05,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:07,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:08,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:47:10,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:13,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:13,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:13,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:13,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:13,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:15,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:15,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:15,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:16,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:16,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:19,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:21,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:22,076 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:24,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:24,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:24,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:24,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:25,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:25,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:26,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:28,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:30,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:30,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:31,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:31,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:34,153 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:34,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:34,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:34,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:34,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:37,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:37,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:37,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:38,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:47:40,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:43,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:43,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:44,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:44,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:44,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:44,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:46,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:46,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:47:46,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:47:48,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:49,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:52,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:53,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:54,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:54,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:54,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:54,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:47:55,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:47:55,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:58,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:47:59,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:47:59,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:47:59,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:47:59,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:47:59,594 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:48:01,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:48:01,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:48:01,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:01,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:04,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:08,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:48:09,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:14,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:16,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:16,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:20,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:25,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:30,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:31,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:31,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:35,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:38,861 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:48:41,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:46,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:48:46,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:48:47,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:52,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:48:54,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_aca4a003a8d6b3b9f87b.png with policy now +2023-07-12 02:48:54,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,165 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_4155d6ba117c33d5628b.png with policy now +2023-07-12 02:48:54,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_65494724a21e7695e782.png with policy now +2023-07-12 02:48:54,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:48:54,167 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_660_3806edfc6615feb4c502.png with policy now +2023-07-12 02:48:54,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:48:54,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:48:54,193 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:48:54,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:48:54,450 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jp0k4gti-media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 02:48:54,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fhru8fuy-media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 02:48:54,502 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6o1k0gzt-media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 02:48:54,530 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/si6981io-media/images/validation_660_65494724a21e7695e782.png +2023-07-12 02:48:55,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:48:55,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 02:48:55,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 02:48:55,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 02:48:55,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_65494724a21e7695e782.png +2023-07-12 02:48:55,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:48:55,744 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/go8azig8-media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 02:48:55,756 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ejf51owy-media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 02:48:55,761 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2d2ncd3n-media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 02:48:55,766 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eniwcuwi-media/images/validation_660_65494724a21e7695e782.png +2023-07-12 02:48:58,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:01,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:01,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:01,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:03,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:04,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:04,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:07,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:08,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:49:08,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:13,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:13,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:13,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:13,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:13,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:13,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:14,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:16,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:17,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:17,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:19,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:19,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:23,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:23,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:23,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:23,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:25,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:25,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:25,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:28,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:30,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:31,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:32,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:32,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:34,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:34,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:34,195 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:34,197 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:34,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:34,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:36,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:37,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:38,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:49:40,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:41,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:43,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:44,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:44,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:44,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:44,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:46,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:47,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:47,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:49:47,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:49:49,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:52,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:52,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:54,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:49:54,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:49:54,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:49:54,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:49:55,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:49:55,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:49:58,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:49:58,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:01,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:02,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:02,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:04,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:04,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:04,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:04,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:04,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:04,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:07,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:07,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:08,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:50:09,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:10,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:13,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:14,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:14,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:14,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:14,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:14,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:17,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:17,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:19,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:19,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:22,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:24,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:24,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:24,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:24,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:25,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:25,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:25,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:28,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:29,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:50:29,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:50:29,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:50:29,344 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:50:30,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:31,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:50:31,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:50:32,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:32,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:36,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:38,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:50:41,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:46,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:47,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:50:47,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:50:52,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:50:57,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:02,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:02,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:03,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:08,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:08,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:51:13,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:17,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:17,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:19,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:23,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_00b528cb36f02641e0c4.png with policy now +2023-07-12 02:51:23,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_4b3b71fb783337ee170d.png with policy now +2023-07-12 02:51:23,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_a54a1c21b8282d62f8f9.png with policy now +2023-07-12 02:51:23,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:51:23,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_670_38a8a2cd9ee27689f5ea.png with policy now +2023-07-12 02:51:23,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:23,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:51:23,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:51:23,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:51:23,880 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2w5ejt9f-media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 02:51:23,943 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vazcpxha-media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 02:51:23,957 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7arzg2g6-media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 02:51:23,971 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ur9xor31-media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 02:51:24,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:25,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:51:25,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 02:51:25,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 02:51:25,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 02:51:25,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 02:51:25,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:51:26,298 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jujya9hd-media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 02:51:26,299 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rd1z54qt-media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 02:51:26,305 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sk0kogsn-media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 02:51:26,308 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyfxv51o-media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 02:51:28,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:30,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:31,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:32,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:32,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:33,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:34,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:35,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:37,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:38,868 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:51:40,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:40,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:43,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:43,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:51:43,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:51:43,156 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:51:43,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:51:43,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:46,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:46,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:47,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:51:47,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:51:49,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:52,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:53,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:51:53,243 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:51:53,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:51:53,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:51:55,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:51:55,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:51:57,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:51:58,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:01,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:02,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:02,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:02,851 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:03,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:03,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:03,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:03,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:04,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:04,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:07,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:08,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:08,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:52:10,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:13,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:13,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:13,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:13,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:13,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:14,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:14,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:17,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:18,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:18,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:18,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:20,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:23,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:23,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:23,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:23,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:23,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:26,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:26,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:29,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:29,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:33,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:33,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:33,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:33,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:33,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:33,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:34,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:35,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:35,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:38,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:38,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:52:39,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:41,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:43,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:43,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:43,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:43,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:44,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:44,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:45,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:47,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:48,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:52:48,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:52:51,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:53,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:53,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:53,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:53,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:53,981 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:56,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:56,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:52:56,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:52:59,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:52:59,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:52:59,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:52:59,026 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:52:59,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:52:59,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:53:02,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:53:02,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:03,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:03,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:07,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:08,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:53:12,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:17,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:18,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:18,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:23,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:28,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:33,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:33,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:34,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:38,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:53:39,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:44,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:48,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:53:48,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:53:50,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:53,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_7d58abcefc10987ef6e0.png with policy now +2023-07-12 02:53:53,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_96b5253932542ff8e0ec.png with policy now +2023-07-12 02:53:53,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_5420d62f64c76dfb3aaf.png with policy now +2023-07-12 02:53:53,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:53:53,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_680_ee4486348a50df1eb1b0.png with policy now +2023-07-12 02:53:53,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:53:53,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:53:53,466 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:53:53,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:53:53,835 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0btltlye-media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 02:53:53,842 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m1pyrm7a-media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 02:53:53,847 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rtmwrw25-media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 02:53:53,877 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hx3cvgrt-media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 02:53:56,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:53:56,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 02:53:56,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 02:53:56,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 02:53:56,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 02:53:56,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:53:56,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:53:56,724 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6fa02k7-media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 02:53:56,733 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/394ck36j-media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 02:53:56,746 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dil9q5xx-media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 02:53:56,795 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kpid1dzq-media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 02:53:59,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:02,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:02,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:03,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:03,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:03,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:05,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:07,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:08,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:08,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:54:11,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:12,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:12,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:12,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:12,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:12,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:14,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:14,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:17,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:17,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:19,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:19,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:20,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:22,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:22,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:22,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:22,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:22,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:23,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:23,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:28,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:29,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:32,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:32,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:32,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:32,742 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:32,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:33,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:34,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:34,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:35,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:35,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:38,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:38,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:54:38,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:41,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:43,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:43,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:43,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:43,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:44,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:44,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:47,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:50,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:50,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:50,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:54:50,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:54:53,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:53,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:54:53,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:54:53,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:54:53,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:54:55,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:54:56,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:54:56,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:54:59,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:01,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:04,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:04,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:04,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:04,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:05,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:05,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:06,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:06,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:06,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:08,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:08,875 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:55:11,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:11,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:14,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:14,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:14,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:14,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:14,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:14,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:16,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:17,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:21,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:21,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:22,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:23,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:25,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:25,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:25,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:25,212 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:26,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:27,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:29,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:29,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:55:29,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:55:29,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:55:29,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:55:32,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:55:32,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:55:33,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:36,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:36,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:38,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:55:38,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:43,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:48,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:51,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:55:51,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:55:54,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:55:59,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:04,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:06,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:06,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:08,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:56:09,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:14,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:19,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:21,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:21,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:24,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,221 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_1ff9e60aee819ba10a7e.png with policy now +2023-07-12 02:56:24,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_8220342bb1673b65f66a.png with policy now +2023-07-12 02:56:24,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,241 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_9749b628dfc772a6cc47.png with policy now +2023-07-12 02:56:24,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:56:24,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_690_dd5fdf92beda60fe8a12.png with policy now +2023-07-12 02:56:24,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:24,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:56:24,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:56:24,293 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:56:24,595 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gwlpzdh2-media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 02:56:24,596 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w4bsddzq-media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 02:56:24,597 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iefdxjb3-media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 02:56:24,632 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zu5v4r6d-media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 02:56:25,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:26,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:56:26,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 02:56:26,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 02:56:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 02:56:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 02:56:26,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:56:27,279 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iwlxkjb5-media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 02:56:27,289 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ulndhypd-media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 02:56:27,324 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9nbebl6u-media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 02:56:27,334 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hdelggno-media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 02:56:29,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:30,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:32,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:34,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:35,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:36,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:36,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:36,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:38,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:56:38,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:41,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:41,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:43,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:43,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:56:43,395 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:56:43,397 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:56:44,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:56:47,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:47,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:50,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:51,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:56:51,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:56:53,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:53,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:56:53,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:56:53,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:56:53,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:56:53,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:56:53,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:56:58,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:56:59,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:03,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:03,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:03,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:03,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:03,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:06,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:06,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:06,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:06,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:08,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:08,879 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:57:09,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:12,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:13,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:13,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:13,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:13,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:13,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:15,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:15,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:18,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:19,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:21,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:21,628 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:24,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:24,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:24,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:24,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:24,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:24,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:27,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:27,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:30,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:30,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:33,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:35,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:35,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:35,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:35,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:36,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:36,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:36,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:37,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:37,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:38,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:57:41,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:42,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:45,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:45,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:45,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:45,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:45,422 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:47,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:48,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:48,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:51,184 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:52,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:57:52,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:57:52,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:57:54,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:57:55,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:57:55,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:57:55,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:57:55,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:57:57,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:57:57,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:00,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:58:00,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:58:00,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:58:00,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:58:00,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:58:00,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:58:03,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:58:03,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:07,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:07,704 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:08,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:08,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:58:13,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:18,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:22,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:22,892 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:23,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:28,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:33,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:38,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:38,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:38,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:58:39,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:44,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:49,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:53,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:58:53,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:58:54,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_002747928010272ad4e0.png with policy now +2023-07-12 02:58:54,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_f85b220903af16d48ab4.png with policy now +2023-07-12 02:58:54,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_ccf6b7bd526cf61a21d8.png with policy now +2023-07-12 02:58:54,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 02:58:54,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_700_11469b6c8fd1d388b8bd.png with policy now +2023-07-12 02:58:54,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:58:54,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:58:54,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:58:54,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:58:55,068 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2fkphk8-media/images/validation_700_002747928010272ad4e0.png +2023-07-12 02:58:55,118 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1rux600b-media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 02:58:55,128 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e5wmjqc7-media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 02:58:55,130 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tf4qihmw-media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 02:58:55,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:58:57,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:58:57,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 02:58:57,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 02:58:57,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_002747928010272ad4e0.png +2023-07-12 02:58:57,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 02:58:57,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 02:58:57,726 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ru4trg10-media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 02:58:57,731 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kf1jvpgj-media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 02:58:57,736 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uinki999-media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 02:58:57,757 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/92i7cdk4-media/images/validation_700_002747928010272ad4e0.png +2023-07-12 02:59:00,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:01,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:03,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:04,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:06,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:06,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:08,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:08,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:08,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:59:09,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:12,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:12,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:13,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:13,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:13,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:13,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:15,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:15,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:18,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:18,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:21,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:23,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:23,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:23,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:24,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:24,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:24,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:24,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:24,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:24,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:27,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:29,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:30,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:33,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:34,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:35,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:35,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:35,208 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:35,210 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:36,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:36,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:38,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:38,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:38,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 02:59:39,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:42,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:45,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:45,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:45,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:45,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:45,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:45,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 02:59:45,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:48,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:50,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:51,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:53,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 02:59:53,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 02:59:54,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 02:59:55,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 02:59:55,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 02:59:55,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 02:59:55,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 02:59:55,700 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 02:59:57,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:00,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:00,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:03,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:06,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:06,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:06,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:06,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:06,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:06,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:06,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:08,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:08,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:08,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:00:09,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:11,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:12,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:15,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:16,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:16,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:16,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:16,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:17,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:18,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:18,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:21,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:23,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:23,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:23,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:24,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:26,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:26,628 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:26,628 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:26,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:27,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:27,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:28,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:30,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:31,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:00:31,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:00:31,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:00:31,536 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:00:33,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:00:33,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:34,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:36,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:00:38,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:38,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:38,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:00:39,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:44,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:49,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:00:53,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:00:53,716 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:00:55,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:00,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:05,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:08,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:08,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:08,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:01:10,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:15,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:20,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:23,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:23,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:25,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,884 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_f8ad001e252a38c302da.png with policy now +2023-07-12 03:01:25,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_e29ac85551bb1427b956.png with policy now +2023-07-12 03:01:25,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:25,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_4e2183818f3e746eee9b.png with policy now +2023-07-12 03:01:25,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:01:25,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_710_42e1f742968073010893.png with policy now +2023-07-12 03:01:25,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:25,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:01:25,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:01:25,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:01:26,188 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wnnp05iy-media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 03:01:26,232 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/au0x0odg-media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 03:01:26,247 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x12ho4sq-media/images/validation_710_42e1f742968073010893.png +2023-07-12 03:01:26,260 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gk4wb1q5-media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 03:01:27,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:01:27,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 03:01:27,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 03:01:27,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_42e1f742968073010893.png +2023-07-12 03:01:27,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 03:01:27,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:01:28,326 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bep6zlhy-media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 03:01:28,327 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lonqkvdj-media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 03:01:28,339 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7w2zz6nr-media/images/validation_710_42e1f742968073010893.png +2023-07-12 03:01:28,353 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8gm4kpk7-media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 03:01:30,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:31,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:33,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:35,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:36,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:37,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:38,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:38,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:38,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:01:40,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:42,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:43,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:45,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:45,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:01:45,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:01:45,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:01:46,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:01:46,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:47,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:49,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:52,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:53,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:01:54,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:01:54,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:01:54,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:01:54,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:01:54,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:01:54,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:01:55,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:01:55,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:58,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:01:58,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:03,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:04,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:04,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:04,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:04,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:04,854 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:07,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:07,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:08,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:08,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:02:09,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:09,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:10,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:13,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:14,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:15,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:15,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:15,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:15,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:16,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:16,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:19,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:22,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:25,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:25,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:25,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:25,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:26,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:26,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:26,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:26,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:28,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:28,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:30,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:31,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:34,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:35,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:35,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:35,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:35,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:35,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:37,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:38,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:02:40,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:40,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:40,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:40,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:43,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:45,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:45,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:45,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:45,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:46,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:46,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:46,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:51,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:52,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:55,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:02:55,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:02:55,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:02:55,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:02:55,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:02:55,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:02:55,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:02:56,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:02:58,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:02:58,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:03:00,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:03:00,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:03:00,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:03:00,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:03:01,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:03:01,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:03:02,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:04,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:03:07,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:08,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:03:10,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:10,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:12,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:17,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:22,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:25,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:25,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:28,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:33,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:38,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:38,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:03:40,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:40,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:43,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:48,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:53,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:03:55,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_0ecdc87dea1892df4eaf.png with policy now +2023-07-12 03:03:55,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_9c940486a7ed24710766.png with policy now +2023-07-12 03:03:55,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_a812b1f440f7f52d6145.png with policy now +2023-07-12 03:03:55,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:03:55,051 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_720_dc11c1be8869686a04d9.png with policy now +2023-07-12 03:03:55,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:03:55,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:03:55,098 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:03:55,104 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:03:55,332 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/loz4wj89-media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 03:03:55,334 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7nmp9aqn-media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 03:03:55,355 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aph4guql-media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 03:03:55,363 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ryrtxmhf-media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 03:03:55,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:03:55,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 03:03:55,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 03:03:55,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 03:03:55,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 03:03:55,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:03:55,805 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgyg7s1u-media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 03:03:55,829 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d4qwpi8k-media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 03:03:55,844 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7wm9c9vv-media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 03:03:55,871 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bfcgc6wp-media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 03:03:55,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:03:55,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:03:59,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:01,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:04,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:05,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:05,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:07,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:08,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:04:10,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:10,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:12,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:12,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:13,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:15,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:15,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:15,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:15,422 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:16,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:16,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:16,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:19,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:22,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:22,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:25,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:25,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:25,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:25,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:25,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:27,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:27,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:27,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:28,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:28,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:31,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:32,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:34,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:36,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:36,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:36,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:36,192 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:37,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:38,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:38,896 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:04:40,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:43,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:43,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:43,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:43,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:46,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:46,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:46,201 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:46,203 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:46,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:46,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:48,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:49,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:52,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:54,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:04:55,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:55,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:04:55,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:04:55,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:04:55,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:04:58,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:04:58,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:04:58,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:04:58,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:04:59,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:01,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:04,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:04,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:06,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:06,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:06,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:06,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:07,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:08,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:05:09,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:10,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:13,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:13,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:05:15,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:16,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:17,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:17,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:17,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:17,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:19,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:19,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:21,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:22,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:25,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:26,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:27,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:27,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:27,260 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:27,262 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:28,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:28,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:29,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:29,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:05:31,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:31,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:05:31,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:31,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:05:31,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:05:31,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:05:34,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:05:34,780 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:05:37,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:38,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:05:42,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:44,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:44,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:05:48,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:53,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:58,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:05:59,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:05:59,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:03,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:08,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:08,900 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:06:13,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:14,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:14,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:19,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:24,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:27,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_61523736602b93efd4f1.png with policy now +2023-07-12 03:06:27,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_b3042d96cd6c0051282b.png with policy now +2023-07-12 03:06:27,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_092236683fae816d1e06.png with policy now +2023-07-12 03:06:27,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:06:27,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_730_0d352eba665388c33ae3.png with policy now +2023-07-12 03:06:27,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:27,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:06:27,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:06:27,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:06:27,493 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cefn0ufl-media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 03:06:27,509 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/czuh5p18-media/images/validation_730_092236683fae816d1e06.png +2023-07-12 03:06:27,529 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b8avwlpz-media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 03:06:27,537 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7hzstif6-media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 03:06:28,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:06:28,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_092236683fae816d1e06.png +2023-07-12 03:06:28,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 03:06:28,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 03:06:28,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 03:06:28,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:06:29,327 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m9mpcfzu-media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 03:06:29,359 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ogyak5wo-media/images/validation_730_092236683fae816d1e06.png +2023-07-12 03:06:29,362 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ckyzeeii-media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 03:06:29,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:29,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:29,437 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/56e3hnkk-media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 03:06:29,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:31,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:34,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:37,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:38,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:38,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:06:40,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:41,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:44,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:44,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:44,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:06:46,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:46,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:46,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:06:46,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:06:46,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:06:47,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:06:47,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:50,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:52,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:53,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:56,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:56,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:06:56,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:06:56,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:06:56,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:06:58,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:06:59,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:06:59,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:06:59,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:06:59,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:02,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:03,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:05,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:06,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:06,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:06,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:06,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:08,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:08,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:07:11,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:13,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:14,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:14,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:07:14,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:16,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:16,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:16,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:16,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:17,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:17,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:18,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:20,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:23,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:23,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:26,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:26,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:26,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:26,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:26,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:29,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:29,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:29,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:29,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:07:29,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:32,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:34,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:35,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:36,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:36,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:36,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:36,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:38,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:38,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:38,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:07:39,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:41,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:44,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:44,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:07:44,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:07:45,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:47,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:47,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:47,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:47,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:47,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:50,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:50,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:50,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:53,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:56,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:07:56,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:07:58,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:07:58,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:07:58,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:07:58,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:07:59,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:07:59,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:08:00,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:00,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:02,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:02,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:08:03,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:08:03,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:08:03,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:08:03,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:08:05,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:08:05,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:08:07,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:08,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:08:12,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:15,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:15,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:17,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:22,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:27,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:30,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:30,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:33,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:38,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:08:39,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:44,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:45,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:08:45,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:08:49,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:54,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:08:58,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_4ca01dfc0ecd320c19a8.png with policy now +2023-07-12 03:08:58,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_472a400ca408d21e03bd.png with policy now +2023-07-12 03:08:58,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_d97239257eb1f6f99304.png with policy now +2023-07-12 03:08:58,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:08:58,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_740_46e887ae6b4e77b77e34.png with policy now +2023-07-12 03:08:58,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:08:58,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:08:58,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:08:58,173 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:08:58,457 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9r2jdwq0-media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 03:08:58,520 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1y09fqz2-media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 03:08:58,528 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1xi6qdo-media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 03:08:58,542 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3kb0h3xg-media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 03:08:59,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:08:59,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 03:08:59,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 03:08:59,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 03:08:59,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 03:08:59,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:08:59,843 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icmv2abo-media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 03:08:59,844 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjftq3t5-media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 03:08:59,865 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9bchwpsf-media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 03:08:59,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/84pgvy6x-media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 03:09:00,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:00,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:00,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:05,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:06,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:07,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:08,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:09:10,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:11,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:14,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:16,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:16,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:16,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:17,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:17,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:17,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:17,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:18,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:18,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:22,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:22,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:26,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:27,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:27,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:27,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:27,273 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:27,275 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:30,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:30,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:31,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:31,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:32,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:34,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:36,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:36,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:36,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:36,625 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:38,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:38,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:38,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:38,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:09:42,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:43,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:46,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:46,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:09:46,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:09:47,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:47,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:47,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:47,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:49,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:50,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:50,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:54,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:09:54,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:57,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:09:57,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:09:57,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:09:57,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:09:58,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:09:58,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:09:59,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:01,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:01,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:02,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:04,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:06,674 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:07,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:07,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:07,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:07,576 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:08,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:10:09,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:10,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:10,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:14,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:15,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:16,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:16,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:18,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:18,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:18,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:18,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:18,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:20,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:22,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:22,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:26,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:26,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:28,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:28,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:28,993 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:28,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:30,731 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:30,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:31,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:31,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:31,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:33,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:10:33,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:10:33,587 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:10:33,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:10:34,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:10:34,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:37,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:38,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:10:38,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:10:42,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:46,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:10:46,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:10:48,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:53,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:10:58,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:01,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:01,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:03,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:08,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:11:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:14,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:16,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:16,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:19,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:24,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:28,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_6a4042dd0f145a5a90ed.png with policy now +2023-07-12 03:11:28,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_d0782df35bd2b2c3cded.png with policy now +2023-07-12 03:11:28,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_f007accf327156196715.png with policy now +2023-07-12 03:11:28,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:11:28,123 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_750_952512d46b1737ec3172.png with policy now +2023-07-12 03:11:28,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:28,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:11:28,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:11:28,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:11:28,380 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pzh3jjk9-media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 03:11:28,442 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mpss25h0-media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 03:11:28,465 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oyxo5jr4-media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 03:11:28,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zc5qvpmt-media/images/validation_750_f007accf327156196715.png +2023-07-12 03:11:30,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:30,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:11:30,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 03:11:30,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 03:11:30,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 03:11:30,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_f007accf327156196715.png +2023-07-12 03:11:30,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:11:31,265 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kke9egdl-media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 03:11:31,273 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mep966ea-media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 03:11:31,274 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1lscod27-media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 03:11:31,301 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/68vwtfl5-media/images/validation_750_f007accf327156196715.png +2023-07-12 03:11:31,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:31,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:34,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:35,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:38,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:38,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:11:38,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:41,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:42,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:46,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:46,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:11:46,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:11:46,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:47,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:47,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:11:47,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:11:47,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:11:50,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:11:50,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:52,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:54,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:11:57,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:11:57,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:11:57,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:11:57,201 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:11:57,203 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:11:58,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:11:58,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:01,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:01,887 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:02,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:02,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:06,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:07,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:07,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:07,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:07,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:07,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:08,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:12:10,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:10,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:13,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:15,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:16,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:16,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:17,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:17,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:17,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:17,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:19,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:19,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:19,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:23,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:24,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:27,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:27,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:27,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:27,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:27,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:29,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:31,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:31,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:32,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:32,159 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:35,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:35,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:37,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:37,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:37,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:37,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:38,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:12:39,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:39,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:41,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:43,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:47,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:47,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:48,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:12:48,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:12:49,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:49,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:49,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:49,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:12:51,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:12:51,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:52,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:55,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:58,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:12:59,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:12:59,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:12:59,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:12:59,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:12:59,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:13:03,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:13:03,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:13:03,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:03,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:03,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:04,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:13:04,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:13:04,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:13:04,042 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:13:07,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:13:07,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:13:08,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:08,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:13:13,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:18,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:19,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:24,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:29,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:33,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:33,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:34,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:38,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:13:39,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:44,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:48,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:13:48,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:13:49,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:54,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:13:58,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_34b823a11a0cf4013af4.png with policy now +2023-07-12 03:13:58,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,354 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_af5422b14a97cc645625.png with policy now +2023-07-12 03:13:58,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,354 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_95d5c0a5c24789a3a475.png with policy now +2023-07-12 03:13:58,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:13:58,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_760_00dc8fdce08e1ae13772.png with policy now +2023-07-12 03:13:58,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:13:58,379 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:13:58,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:13:58,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:13:58,639 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gubehukp-media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 03:13:58,678 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rr2150nl-media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 03:13:58,693 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mrz4vr5o-media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 03:13:58,702 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8lz8sef9-media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 03:13:59,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:13:59,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 03:13:59,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 03:13:59,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 03:13:59,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 03:13:59,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:13:59,672 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/97ygv7ud-media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 03:13:59,673 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xlu2o9uc-media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 03:13:59,697 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mg6s6h5q-media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 03:13:59,790 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ef50zv5i-media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 03:14:00,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:03,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:03,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:03,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:05,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:07,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:07,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:08,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:14:11,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:11,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:15,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:17,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:17,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:17,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:17,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:17,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:19,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:19,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:19,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:19,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:22,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:23,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:27,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:27,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:27,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:27,260 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:27,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:27,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:27,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:31,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:32,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:35,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:35,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:35,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:37,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:38,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:38,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:38,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:38,196 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:38,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:14:39,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:39,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:43,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:43,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:47,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:47,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:47,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:47,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:47,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:49,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:50,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:14:50,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:14:51,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:51,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:54,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:14:55,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:14:58,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:14:58,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:14:58,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:14:58,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:14:59,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:14:59,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:00,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:03,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:05,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:06,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:06,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:07,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:08,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:15:09,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:09,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:09,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:09,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:11,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:11,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:11,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:15,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:16,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:19,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:19,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:19,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:19,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:19,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:19,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:21,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:21,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:21,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:23,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:26,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:27,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:29,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:29,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:29,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:29,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:31,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:31,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:32,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:34,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:15:34,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:15:34,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:15:34,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:15:35,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:15:35,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:15:36,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:36,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:39,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:15:39,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:44,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:49,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:15:51,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:15:51,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:15:55,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:00,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:05,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:06,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:06,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:08,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:16:10,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:15,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:20,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:21,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:21,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:28,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,465 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_df68bb8ae7cb3ad86921.png with policy now +2023-07-12 03:16:28,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png with policy now +2023-07-12 03:16:28,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_cba209758498588b8517.png with policy now +2023-07-12 03:16:28,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:16:28,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_770_217409d7c0158fc1dcec.png with policy now +2023-07-12 03:16:28,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:28,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:16:28,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:16:28,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:16:28,771 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cykuedw8-media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 03:16:28,813 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ijcx89sw-media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 03:16:28,834 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eylk86jg-media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 03:16:28,864 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3lcd60q-media/images/validation_770_cba209758498588b8517.png +2023-07-12 03:16:31,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:31,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:16:31,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 03:16:31,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_cba209758498588b8517.png +2023-07-12 03:16:31,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 03:16:31,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 03:16:31,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:16:32,157 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m77gpaq5-media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 03:16:32,163 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ek1jd4lq-media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 03:16:32,164 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r38pt0rd-media/images/validation_770_cba209758498588b8517.png +2023-07-12 03:16:32,166 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbl9urib-media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 03:16:35,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:36,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:36,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:36,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:38,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:16:39,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:39,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:42,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:43,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:47,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:48,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:48,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:48,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:16:48,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:16:48,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:16:51,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:16:51,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:16:51,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:16:51,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:53,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:55,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:16:58,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:16:58,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:16:58,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:16:58,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:16:59,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:16:59,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:16:59,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:03,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:04,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:07,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:07,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:07,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:07,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:07,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:07,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:07,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:07,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:08,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:17:09,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:11,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:14,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:15,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:17,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:17,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:17,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:17,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:19,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:19,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:20,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:22,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:22,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:23,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:26,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:27,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:27,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:27,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:27,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:27,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:27,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:31,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:31,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:35,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:36,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:37,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:37,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:37,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:37,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:37,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:37,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:38,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:17:39,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:39,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:41,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:43,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:47,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:47,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:49,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:49,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:49,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:49,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:17:52,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:17:52,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:52,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:17:52,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:52,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:17:56,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:17:57,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:17:59,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:17:59,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:17:59,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:17:59,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:18:00,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:18:00,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:18:03,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:04,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:18:04,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:18:04,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:18:04,525 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:18:04,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:18:07,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:07,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:08,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:18:08,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:18:08,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:18:08,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:13,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:18,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:22,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:22,503 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:24,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:29,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:34,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:37,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:37,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:38,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:18:39,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:44,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:49,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:52,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:18:52,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:18:55,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:18:58,729 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,729 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_5bc6ddbcab2869abe354.png with policy now +2023-07-12 03:18:58,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,769 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_8cd3d57f7ce32e8a2128.png with policy now +2023-07-12 03:18:58,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_8cb1b948d1d2b956831d.png with policy now +2023-07-12 03:18:58,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:18:58,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_780_6240b2c0fb96b2b24cb6.png with policy now +2023-07-12 03:18:58,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:18:58,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:18:58,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:18:58,805 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:18:59,080 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rddwit8k-media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 03:18:59,081 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qrgv0lb9-media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 03:18:59,090 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cbs7v4hx-media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 03:18:59,095 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bpg9h7d5-media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 03:19:00,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:00,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 03:19:00,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 03:19:00,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 03:19:00,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 03:19:00,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:19:00,568 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7lhv85g4-media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 03:19:00,578 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0c51lmqb-media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 03:19:00,586 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02qntig4-media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 03:19:00,604 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/umgsg1je-media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 03:19:00,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:04,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:06,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:07,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:07,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:08,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:08,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:19:08,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:12,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:12,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:16,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:17,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:17,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:17,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:17,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:17,903 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:20,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:20,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:22,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:22,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:23,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:24,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:27,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:27,833 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:27,833 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:27,835 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:28,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:28,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:28,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:32,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:34,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:36,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:37,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:37,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:38,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:38,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:38,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:38,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:38,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:19:39,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:40,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:40,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:44,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:48,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:49,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:19:49,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:19:49,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:19:49,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:19:51,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:19:52,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:19:52,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:52,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:19:52,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:19:56,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:19:56,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:00,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:00,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:00,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:00,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:00,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:00,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:02,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:04,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:07,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:07,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:07,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:08,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:08,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:20:10,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:10,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:10,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:10,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:12,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:12,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:12,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:16,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:17,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:20,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:21,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:21,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:21,502 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:21,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:23,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:23,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:23,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:24,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:24,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:28,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:28,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:31,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:31,861 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:31,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:31,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:32,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:32,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:33,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:36,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:20:36,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:20:36,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:20:36,374 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:20:36,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:20:36,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:38,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:38,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:38,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:20:38,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:40,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:20:43,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:48,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:53,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:20:53,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:20:54,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:20:59,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:04,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:08,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:08,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:08,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:21:09,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:14,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:19,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:23,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:23,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:25,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:30,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:30,832 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_f18cdbdebab2bd2197d4.png with policy now +2023-07-12 03:21:30,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_7c032ae6c9675c498efb.png with policy now +2023-07-12 03:21:30,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_0e3bece4b24d9f0cff63.png with policy now +2023-07-12 03:21:30,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:21:30,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_790_2c76661900a84617ca31.png with policy now +2023-07-12 03:21:30,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:21:30,907 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:21:30,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:21:30,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:21:31,135 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wohrxp4c-media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 03:21:31,166 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ma4qfgwq-media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 03:21:31,170 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ldbwo4n9-media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 03:21:31,187 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4bxfx133-media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 03:21:32,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:21:32,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 03:21:32,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 03:21:32,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 03:21:32,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 03:21:32,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:21:33,058 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhzt10zx-media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 03:21:33,074 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dmamb09u-media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 03:21:33,096 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ubiggxtu-media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 03:21:33,112 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q7ty2fyh-media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 03:21:35,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:36,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:38,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:38,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:38,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:21:40,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:40,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:41,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:21:44,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:46,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:48,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:50,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:21:50,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:21:50,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:21:50,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:21:52,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:21:52,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:21:52,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:53,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:21:53,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:21:56,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:21:57,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:00,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:00,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:00,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:00,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:00,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:00,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:02,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:04,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:07,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:08,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:08,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:08,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:08,935 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:22:11,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:11,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:11,118 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:11,120 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:12,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:12,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:13,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:16,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:18,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:20,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:21,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:21,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:21,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:21,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:23,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:23,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:23,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:24,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:24,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:28,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:28,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:31,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:31,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:31,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:31,229 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:32,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:32,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:34,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:36,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:38,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:38,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:38,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:22:39,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:41,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:42,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:42,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:42,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:42,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:45,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:45,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:45,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:49,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:50,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:52,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:22:52,977 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:22:52,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:22:52,979 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:22:53,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:22:53,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:22:53,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:22:53,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:22:56,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:22:57,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:01,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:01,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:02,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:23:02,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:23:02,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:23:02,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:23:05,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:23:05,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:07,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:07,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:23:07,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:23:07,275 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:23:07,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:23:08,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:08,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:08,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:23:09,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:23:09,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:12,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:13,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:23:17,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:22,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:23,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:23,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:28,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:33,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:38,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:38,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:38,579 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:38,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:23:43,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:48,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:53,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:23:53,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:23:54,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:23:59,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:01,323 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_a66b82318049cd1e171c.png with policy now +2023-07-12 03:24:01,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_ab1fea6190aed6bcea40.png with policy now +2023-07-12 03:24:01,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,366 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_c3dd46dd047d49388d6c.png with policy now +2023-07-12 03:24:01,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:24:01,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_800_9c63a115f12e85651901.png with policy now +2023-07-12 03:24:01,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:01,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:01,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:01,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:01,643 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y86rhgl1-media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 03:24:01,682 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/thtiht63-media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 03:24:01,697 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2ln7hpi-media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 03:24:01,701 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7x7gie5-media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 03:24:05,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 03:24:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 03:24:05,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 03:24:05,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 03:24:05,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:24:05,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:05,616 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nhvc10yb-media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 03:24:05,630 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0glcmc9z-media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 03:24:05,638 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8h0rlkvj-media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 03:24:05,648 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qq84huyp-media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 03:24:08,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:08,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:08,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:24:09,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:11,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:11,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:13,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:16,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:17,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:21,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:21,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:22,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:22,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:22,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:22,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:23,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:23,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:25,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:25,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:27,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:29,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:32,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:32,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:32,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:32,076 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:33,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:33,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:33,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:37,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:38,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:38,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:39,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:24:39,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:41,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:41,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:41,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:41,788 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:41,791 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:45,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:45,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:45,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:49,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:50,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:51,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:24:51,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:24:51,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:24:51,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:24:53,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:24:53,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:24:53,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:24:53,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:24:55,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:24:57,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:00,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:01,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:01,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:01,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:01,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:01,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:05,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:05,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:06,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:08,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:08,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:08,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:25:09,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:11,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:11,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:11,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:11,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:11,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:13,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:13,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:16,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:17,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:21,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:21,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:21,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:21,996 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:21,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:21,999 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:23,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:23,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:25,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:25,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:27,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:29,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:32,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:32,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:32,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:32,788 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:32,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:33,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:33,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:37,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:37,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:25:37,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:25:37,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:25:37,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:25:38,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:38,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:25:38,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:38,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:41,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:25:41,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:25:44,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:25:53,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:25:53,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:25:54,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:00,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:08,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:26:08,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:08,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:10,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:15,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:20,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:23,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:23,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:25,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:30,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:32,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_9a62636574ba68d8fc83.png with policy now +2023-07-12 03:26:32,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_8d25802cfd68926c4540.png with policy now +2023-07-12 03:26:32,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_14a8076e3481bc7a8090.png with policy now +2023-07-12 03:26:32,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:26:32,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_810_3d586403ca68c8dd89ef.png with policy now +2023-07-12 03:26:32,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:26:32,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:26:32,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:26:32,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:26:32,455 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s9t517t5-media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 03:26:32,473 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7c6198ro-media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 03:26:32,483 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oio1uc1i-media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 03:26:32,495 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lccyq7ou-media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 03:26:33,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:26:33,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 03:26:33,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 03:26:33,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 03:26:33,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 03:26:33,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:26:34,031 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tep2ovo9-media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 03:26:34,048 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lmqxju77-media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 03:26:34,049 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0c5xolym-media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 03:26:34,058 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pclu2lri-media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 03:26:35,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:37,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:38,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:26:38,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:38,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:40,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:41,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:42,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:26:45,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:45,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:49,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:50,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:52,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:26:52,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:26:52,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:26:52,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:26:53,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:26:53,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:26:53,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:26:53,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:26:56,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:26:57,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:01,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:01,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:02,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:02,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:02,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:02,200 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:05,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:05,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:07,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:08,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:27:09,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:09,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:09,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:12,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:12,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:12,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:12,750 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:12,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:13,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:13,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:17,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:18,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:21,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:22,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:22,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:22,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:22,479 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:24,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:24,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:24,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:25,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:25,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:29,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:29,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:32,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:32,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:32,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:32,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:33,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:33,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:34,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:37,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:38,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:27:39,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:39,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:41,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:41,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:42,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:42,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:42,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:42,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:45,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:45,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:46,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:49,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:52,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:53,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:27:53,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:27:53,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:27:53,136 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:27:53,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:27:53,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:27:55,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:27:55,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:27:57,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:27:57,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:01,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:03,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:28:03,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:03,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:28:03,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:28:03,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:28:05,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:28:05,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:08,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:28:08,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:28:08,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:28:08,177 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:28:08,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:08,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:28:09,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:28:09,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:10,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:10,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:13,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:28:14,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:19,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:24,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:25,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:25,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:29,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:34,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:38,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:28:39,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:40,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:40,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:45,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:50,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:28:55,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:28:55,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:28:56,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:01,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:02,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_9eeeace7edf859719f14.png with policy now +2023-07-12 03:29:02,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,705 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_498f985682f98f0b515d.png with policy now +2023-07-12 03:29:02,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_c37a81321646db9654fb.png with policy now +2023-07-12 03:29:02,716 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:29:02,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_820_de63bd88f0f3c4502afa.png with policy now +2023-07-12 03:29:02,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:02,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:02,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:02,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:02,981 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yygxsyx0-media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 03:29:03,028 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/akju31dt-media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 03:29:03,068 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rhbf86er-media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 03:29:03,127 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8ee01sa-media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 03:29:06,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:06,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 03:29:06,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 03:29:06,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 03:29:06,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 03:29:06,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:29:06,523 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y08j9pma-media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 03:29:06,536 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pgtr06pp-media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 03:29:06,553 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/048c6o36-media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 03:29:06,561 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k3owf84e-media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 03:29:06,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:08,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:29:10,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:10,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:10,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:12,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:13,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:14,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:17,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:18,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:22,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:22,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:22,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:22,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:22,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:23,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:25,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:25,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:26,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:26,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:29,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:30,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:31,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:31,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:31,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:31,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:34,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:34,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:34,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:38,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:38,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:29:40,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:40,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:40,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:41,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:41,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:41,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:41,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:42,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:42,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:45,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:46,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:50,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:50,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:52,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:29:52,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:29:52,193 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:29:52,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:29:54,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:29:54,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:29:55,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:29:55,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:29:56,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:29:58,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:02,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:02,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:02,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:02,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:02,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:02,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:06,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:06,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:07,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:08,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:30:10,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:10,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:10,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:12,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:12,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:12,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:12,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:13,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:14,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:14,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:18,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:19,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:22,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:22,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:22,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:22,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:22,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:24,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:25,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:25,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:26,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:26,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:30,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:30,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:33,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:33,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:33,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:33,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:34,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:34,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:35,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:38,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:38,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:30:38,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:30:38,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:30:38,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:30:38,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:30:40,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:40,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:41,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:42,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:30:42,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:30:46,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:51,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:30:55,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:30:55,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:30:56,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:01,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:06,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:08,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:31:10,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:10,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:11,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:16,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:21,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:25,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:25,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:26,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:31,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:33,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_5d081a2a0528a30e8ce2.png with policy now +2023-07-12 03:31:33,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_0aca2d7047838c676be5.png with policy now +2023-07-12 03:31:33,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_3bdbbfd9d4a0076d30db.png with policy now +2023-07-12 03:31:33,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:31:33,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_830_73577e16f9e832867bc6.png with policy now +2023-07-12 03:31:33,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:31:33,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:31:33,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:31:33,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:31:33,725 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m8sesui5-media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 03:31:34,238 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/krcz232f-media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 03:31:34,324 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n7vs1mha-media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 03:31:34,443 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fndsi8k1-media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 03:31:34,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:31:34,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 03:31:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 03:31:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 03:31:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 03:31:34,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:31:35,024 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j0hkuzp1-media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 03:31:35,068 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xq54jwsm-media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 03:31:35,071 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k4e0ddv0-media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 03:31:35,460 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kksr49tt-media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 03:31:36,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:38,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:38,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:31:41,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:41,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:42,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:42,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:43,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:31:46,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:47,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:50,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:53,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:31:53,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:53,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:31:53,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:31:53,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:31:54,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:31:54,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:31:56,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:31:56,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:31:58,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:31:58,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:02,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:02,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:02,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:02,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:02,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:03,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:06,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:08,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:32:08,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:10,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:11,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:11,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:13,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:13,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:13,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:13,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:14,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:14,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:14,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:18,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:20,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:22,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:24,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:24,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:24,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:24,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:25,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:26,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:26,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:26,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:26,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:30,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:31,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:34,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:34,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:34,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:34,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:34,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:34,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:36,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:38,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:38,959 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:32:41,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:41,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:42,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:42,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:44,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:44,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:44,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:44,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:46,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:46,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:47,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:50,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:53,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:54,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:32:56,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:32:56,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:32:56,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:32:56,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:32:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:32:56,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:32:58,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:32:58,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:32:58,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:02,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:04,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:06,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:33:06,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:33:06,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:33:06,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:33:06,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:33:06,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:08,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:33:09,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:10,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:33:10,716 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:33:10,716 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:33:10,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:33:10,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:33:10,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:11,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:11,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:33:14,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:33:15,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:20,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:25,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:26,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:26,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:33:30,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:35,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:38,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:33:40,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:41,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:41,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:33:45,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:50,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:55,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:33:57,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:33:57,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:01,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:05,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_90e1e0a7b7d300559299.png with policy now +2023-07-12 03:34:05,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_3cc72b352f6a24b1d731.png with policy now +2023-07-12 03:34:05,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_924f6a01834fef3e5df9.png with policy now +2023-07-12 03:34:05,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:34:05,377 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_840_8376c90a668eab0df0f1.png with policy now +2023-07-12 03:34:05,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:05,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:05,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:05,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:05,658 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bu54zc7x-media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 03:34:05,666 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zc6cgqp9-media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 03:34:05,676 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gbnuoeu3-media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 03:34:05,690 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s9i8iiyj-media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 03:34:06,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:07,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 03:34:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 03:34:07,167 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 03:34:07,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 03:34:07,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:34:07,527 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sfsptab2-media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 03:34:07,528 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mmrqxjlv-media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 03:34:07,538 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4anwj15k-media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 03:34:07,540 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rt1ipyit-media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 03:34:08,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:34:11,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:12,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:12,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:12,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:15,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:15,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:17,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:19,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:23,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:23,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:24,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:24,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:24,249 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:24,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:27,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:27,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:28,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:28,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:28,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:31,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:33,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:33,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:33,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:33,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:34,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:35,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:35,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:38,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:34:39,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:39,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:43,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:43,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:43,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:43,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:43,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:43,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:43,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:45,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:47,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:47,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:50,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:51,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:53,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:34:53,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:34:53,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:34:53,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:34:55,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:34:55,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:34:55,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:34:58,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:34:58,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:34:59,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:00,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:03,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:03,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:03,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:03,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:03,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:06,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:07,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:07,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:08,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:35:11,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:11,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:13,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:13,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:35:14,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:14,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:14,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:14,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:15,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:15,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:16,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:19,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:22,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:23,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:25,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:25,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:25,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:25,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:27,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:27,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:27,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:29,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:29,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:35:31,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:33,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:34,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:34,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:34,982 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:34,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:35,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:35,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:38,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:35:38,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:39,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:39,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:35:39,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:35:39,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:35:39,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:35:43,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:35:43,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:35:44,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:44,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:44,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:35:49,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:54,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:59,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:35:59,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:35:59,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:04,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:08,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:36:09,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:14,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:14,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:15,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:20,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:25,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:29,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:29,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:30,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:34,440 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,440 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_d8f23be87659ed0cbfb4.png with policy now +2023-07-12 03:36:34,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_20f5209698af0ab0b778.png with policy now +2023-07-12 03:36:34,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_20c9060ffc3bdad4a934.png with policy now +2023-07-12 03:36:34,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:36:34,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_850_1f72867e944a10d3e520.png with policy now +2023-07-12 03:36:34,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:36:34,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:36:34,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:36:34,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:36:34,865 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/79x5z2yb-media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 03:36:34,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gpozjq6b-media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 03:36:34,902 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nnu0aab8-media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 03:36:34,934 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7i5zwlkv-media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 03:36:35,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:36:35,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 03:36:35,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 03:36:35,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 03:36:35,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 03:36:35,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:36:36,120 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x8h7sd23-media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 03:36:36,137 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uxacb6z3-media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 03:36:36,161 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6xskmv1h-media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 03:36:36,193 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wji6g6tp-media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 03:36:36,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:38,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:36:39,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:42,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:43,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:44,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:36:44,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:44,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:36:47,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:47,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:51,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:53,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:54,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:36:54,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:36:54,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:36:54,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:36:55,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:36:55,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:58,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:36:59,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:36:59,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:36:59,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:03,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:03,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:04,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:04,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:04,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:04,491 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:07,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:07,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:08,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:37:08,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:11,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:14,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:14,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:37:14,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:14,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:14,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:14,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:14,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:15,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:15,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:19,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:19,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:23,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:25,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:25,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:25,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:25,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:25,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:27,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:27,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:29,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:37:29,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:30,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:31,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:35,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:35,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:35,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:35,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:35,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:35,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:35,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:38,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:37:39,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:41,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:43,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:44,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:37:44,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:37:45,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:45,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:45,445 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:45,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:46,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:47,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:47,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:51,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:51,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:55,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:37:55,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:37:55,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:37:55,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:37:55,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:37:55,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:37:57,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:37:59,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:00,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:00,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:03,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:03,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:05,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:38:05,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:38:05,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:38:05,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:38:07,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:38:07,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:08,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:38:08,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:10,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:38:10,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:38:10,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:38:10,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:38:11,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:38:11,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:38:14,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:15,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:15,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:20,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:25,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:30,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:30,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:31,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:36,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:38,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:38:41,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:45,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:38:45,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:38:47,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:52,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:38:57,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:00,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:00,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:03,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:04,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_97fe22a2e02d107e0669.png with policy now +2023-07-12 03:39:04,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,619 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_8062bd4106d50ccb836d.png with policy now +2023-07-12 03:39:04,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_578c282df31c9a089068.png with policy now +2023-07-12 03:39:04,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:39:04,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_860_1ff63456305716e15a0f.png with policy now +2023-07-12 03:39:04,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:04,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:04,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:04,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:04,967 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1s2mjyqo-media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 03:39:04,980 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/imalsj4f-media/images/validation_860_578c282df31c9a089068.png +2023-07-12 03:39:05,020 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/04tfg4q6-media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 03:39:05,086 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ccm5qmt3-media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 03:39:08,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:08,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:08,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 03:39:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 03:39:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_578c282df31c9a089068.png +2023-07-12 03:39:08,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 03:39:08,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:39:08,556 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v6pyph4t-media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 03:39:08,633 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5l3ofxxi-media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 03:39:08,642 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rzrj8et1-media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 03:39:08,689 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qkciqu03-media/images/validation_860_578c282df31c9a089068.png +2023-07-12 03:39:08,976 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:39:08,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:12,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:14,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:14,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:15,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:15,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:16,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:19,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:20,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:23,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:23,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:23,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:23,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:24,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:24,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:24,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:28,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:30,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:30,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:30,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:32,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:33,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:33,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:33,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:33,044 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:35,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:36,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:36,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:38,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:39:40,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:40,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:43,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:43,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:43,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:43,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:44,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:44,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:45,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:39:45,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:39:46,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:48,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:52,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:39:52,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:53,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:39:53,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:39:53,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:39:53,552 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:39:56,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:39:56,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:39:57,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:00,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:01,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:01,232 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:02,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:04,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:04,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:04,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:04,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:04,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:08,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:08,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:08,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:08,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:40:12,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:13,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:14,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:14,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:14,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:14,567 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:16,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:16,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:16,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:16,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:18,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:20,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:24,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:24,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:24,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:24,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:24,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:24,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:28,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:28,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:29,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:32,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:32,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:32,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:34,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:34,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:34,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:34,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:34,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:36,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:36,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:38,981 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:40:39,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:40,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:40:40,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:40:40,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:40:40,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:40:40,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:40:40,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:44,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:40:45,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:47,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:40:47,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:40:51,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:40:56,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:01,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:02,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:02,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:06,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:08,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:41:11,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:16,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:17,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:17,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:22,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:27,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:32,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:32,283 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:32,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:34,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_b6004993162c340948e4.png with policy now +2023-07-12 03:41:34,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_1e7e11e8737344ca3c57.png with policy now +2023-07-12 03:41:34,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_4f7e5756a84f41b58f03.png with policy now +2023-07-12 03:41:34,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:41:34,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_870_674d99e976453deac9a0.png with policy now +2023-07-12 03:41:34,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:41:34,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:41:34,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:41:34,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:41:34,677 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yjhaez30-media/images/validation_870_b6004993162c340948e4.png +2023-07-12 03:41:34,738 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sincgc7j-media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 03:41:34,871 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pamxhpxv-media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 03:41:34,899 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t891uilu-media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 03:41:36,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:41:36,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 03:41:36,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 03:41:36,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 03:41:36,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_b6004993162c340948e4.png +2023-07-12 03:41:36,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:41:36,984 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oubo216i-media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 03:41:37,109 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5ebnwx9n-media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 03:41:37,164 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vgofr6ln-media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 03:41:37,169 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hm9uhm8s-media/images/validation_870_b6004993162c340948e4.png +2023-07-12 03:41:38,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:38,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:41:40,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:43,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:44,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:41:44,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:47,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:41:47,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:41:48,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:49,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:52,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:41:52,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:41:52,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:41:52,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:41:52,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:41:54,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:41:56,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:41:56,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:00,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:00,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:01,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:01,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:01,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:01,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:02,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:02,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:04,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:04,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:06,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:08,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:08,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:42:11,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:11,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:11,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:11,655 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:12,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:12,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:12,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:16,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:17,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:17,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:18,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:20,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:22,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:22,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:22,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:22,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:24,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:24,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:24,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:28,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:29,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:32,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:32,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:32,535 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:32,536 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:32,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:32,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:32,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:32,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:34,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:37,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:38,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:42:39,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:41,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:43,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:43,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:43,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:43,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:45,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:45,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:45,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:47,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:42:47,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:42:49,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:50,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:53,088 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:42:53,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:42:53,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:42:53,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:42:53,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:42:55,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:42:57,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:42:57,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:01,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:01,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:02,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:02,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:43:05,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:43:05,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:43:05,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:43:05,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:43:05,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:07,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:08,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:43:09,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:09,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:43:09,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:43:09,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:43:09,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:43:12,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:13,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:43:13,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:43:17,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:17,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:18,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:23,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:28,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:32,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:32,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:34,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:38,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:43:39,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:44,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:48,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:43:48,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:43:50,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:43:55,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:00,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:03,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:03,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:03,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_c05827ace7407dafe67f.png with policy now +2023-07-12 03:44:03,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_ddb13591ebb5fc4093e5.png with policy now +2023-07-12 03:44:03,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_1a93cee3c2fe9238634b.png with policy now +2023-07-12 03:44:03,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:44:03,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_880_8a9698cc7d3b8724fb2b.png with policy now +2023-07-12 03:44:03,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:03,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:03,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:03,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:03,867 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q3mam0su-media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 03:44:03,869 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bixqksfj-media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 03:44:03,885 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhot14m0-media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 03:44:03,909 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d5egy65t-media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 03:44:05,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:05,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 03:44:05,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 03:44:05,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 03:44:05,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 03:44:05,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:44:05,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:05,699 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/47kf4l6b-media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 03:44:05,702 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/32hzjj9w-media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 03:44:05,744 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gzkec77v-media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 03:44:05,812 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uvox1gun-media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 03:44:08,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:44:09,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:10,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:13,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:13,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:15,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:17,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:18,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:18,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:21,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:21,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:22,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:22,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:22,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:22,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:25,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:25,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:29,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:32,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:32,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:32,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:32,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:32,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:33,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:33,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:33,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:33,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:37,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:38,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:38,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:44:41,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:42,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:42,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:42,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:42,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:43,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:45,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:45,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:49,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:49,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:44:49,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:44:49,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:51,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:44:51,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:44:51,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:44:51,976 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:44:53,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:44:53,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:44:54,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:44:57,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:00,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:01,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:03,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:03,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:03,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:03,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:04,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:04,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:05,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:05,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:05,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:08,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:45:09,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:10,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:13,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:13,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:13,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:13,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:13,745 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:16,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:17,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:17,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:19,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:19,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:21,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:21,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:24,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:24,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:24,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:24,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:25,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:25,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:26,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:29,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:32,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:33,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:34,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:34,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:34,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:34,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:34,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:34,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:37,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:37,575 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:37,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:38,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:45:38,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:45:39,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:45:39,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:45:39,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:45:41,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:45:41,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:45:43,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:48,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:49,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:45:49,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:45:54,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:45:59,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:04,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:04,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:04,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:08,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:46:09,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:14,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:19,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:19,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:20,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:25,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:30,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:33,652 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_f18e48cd66c93df800cc.png with policy now +2023-07-12 03:46:33,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_5f8815ecaca9bc2b8ad7.png with policy now +2023-07-12 03:46:33,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,694 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_01b5da4e3887f97a0716.png with policy now +2023-07-12 03:46:33,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:46:33,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_890_e9f5b2cb5a42a51a191f.png with policy now +2023-07-12 03:46:33,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:46:33,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:46:33,723 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:46:33,725 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:46:33,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:46:33,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 03:46:33,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 03:46:33,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 03:46:33,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 03:46:33,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:46:34,032 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uttiziye-media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 03:46:34,037 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rvh68cir-media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 03:46:34,043 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w2jlwyia-media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 03:46:34,200 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e6g6006d-media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 03:46:34,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:34,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:34,512 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nwtg00jw-media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 03:46:34,578 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h3hbtcz2-media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 03:46:34,579 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tp57ewhw-media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 03:46:34,753 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/edzsiwpp-media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 03:46:35,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:38,996 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:46:40,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:41,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:43,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:46:45,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:46,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:49,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:46:49,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:46:49,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:51,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:53,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:46:53,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:46:53,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:46:53,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:46:53,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:46:53,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:46:57,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:46:57,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:01,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:02,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:02,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:02,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:02,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:02,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:04,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:04,563 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:05,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:05,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:07,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:08,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:47:09,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:13,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:13,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:13,364 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:13,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:13,366 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:13,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:13,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:17,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:19,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:20,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:20,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:21,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:24,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:24,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:24,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:24,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:24,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:25,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:25,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:29,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:29,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:34,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:34,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:34,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:34,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:34,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:34,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:35,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:35,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:38,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:38,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:38,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:47:40,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:42,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:44,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:44,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:44,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:44,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:46,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:46,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:46,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:50,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:51,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:47:51,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:51,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:47:54,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:47:54,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:47:54,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:47:54,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:47:54,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:47:56,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:47:58,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:47:58,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:01,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:02,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:05,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:48:05,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:48:05,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:48:05,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:48:06,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:48:06,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:07,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:07,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:07,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:09,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:48:09,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:48:09,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:48:09,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:48:09,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:48:10,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:48:10,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:12,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:14,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:48:17,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:22,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:22,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:23,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:28,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:33,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:37,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:37,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:39,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:48:39,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:44,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:49,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:52,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:48:52,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:48:54,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:48:59,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:03,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_6956b8efd11196c8a652.png with policy now +2023-07-12 03:49:03,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,846 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_5741bd49ef7880023ce2.png with policy now +2023-07-12 03:49:03,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,846 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_54eb3287687fb6f3d07f.png with policy now +2023-07-12 03:49:03,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:49:03,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_900_4bf5e195f1228d67ca30.png with policy now +2023-07-12 03:49:03,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:03,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:03,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:03,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:04,153 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wvkl55sr-media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 03:49:04,275 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhay3qdf-media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 03:49:04,350 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t40saclh-media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 03:49:04,437 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpvbueie-media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 03:49:04,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:06,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:06,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 03:49:06,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 03:49:06,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 03:49:06,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 03:49:06,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:49:06,647 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qwu9gztl-media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 03:49:06,711 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9gqscbkg-media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 03:49:06,791 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8pt5gvyt-media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 03:49:06,871 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lkhguiws-media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 03:49:07,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:07,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:09,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:49:10,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:10,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:14,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:14,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:16,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:18,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:21,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:22,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:22,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:23,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:23,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:23,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:23,517 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:26,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:26,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:27,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:30,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:32,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:34,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:34,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:34,032 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:34,034 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:34,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:34,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:38,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:38,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:38,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:38,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:39,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:49:42,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:43,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:44,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:44,028 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:44,028 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:44,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:46,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:46,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:49,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:50,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:53,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:49:53,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:49:54,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:54,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:49:54,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:49:54,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:49:54,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:49:54,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:49:58,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:49:58,414 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:49:59,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:02,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:04,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:04,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:04,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:04,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:05,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:06,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:06,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:09,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:50:09,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:09,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:10,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:10,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:14,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:15,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:15,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:15,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:15,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:16,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:18,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:18,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:21,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:22,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:24,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:24,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:25,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:25,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:25,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:25,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:26,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:26,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:26,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:30,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:32,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:34,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:35,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:35,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:35,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:35,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:37,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:38,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:38,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:39,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:50:39,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:50:39,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:50:39,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:50:39,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:50:39,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:39,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:42,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:50:42,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:50:43,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:48,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:53,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:50:54,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:50:54,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:50:59,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:04,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:09,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:51:09,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:09,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:10,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:15,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:20,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:25,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:25,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:25,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:30,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:33,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:33,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_f56c6743f3df48566c7e.png with policy now +2023-07-12 03:51:34,033 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:34,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_85b1581a7259fb0cc926.png with policy now +2023-07-12 03:51:34,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:34,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_4bf762954166a9bf2bce.png with policy now +2023-07-12 03:51:34,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:51:34,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_910_2e5e7f759b91d9be266a.png with policy now +2023-07-12 03:51:34,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:51:34,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:51:34,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:51:34,109 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:51:34,372 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hnfz50f2-media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 03:51:34,374 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w3dy1cfx-media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 03:51:34,420 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2p6rjvg-media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 03:51:34,445 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nohdn0w4-media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 03:51:34,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:51:34,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 03:51:34,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 03:51:34,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 03:51:34,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 03:51:34,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:51:35,237 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yu3i2tfe-media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 03:51:35,264 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j893bq0w-media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 03:51:35,364 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2mq97okc-media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 03:51:35,382 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hvc9t1eg-media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 03:51:36,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:38,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:39,008 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:51:40,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:40,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:41,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:42,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:44,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:51:46,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:46,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:50,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:52,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:54,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:51:54,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:51:54,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:51:54,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:51:54,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:51:54,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:51:55,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:51:55,731 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:51:57,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:51:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:02,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:02,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:04,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:04,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:04,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:04,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:06,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:06,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:07,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:09,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:52:10,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:11,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:11,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:13,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:14,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:14,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:14,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:14,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:14,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:14,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:18,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:18,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:22,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:24,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:25,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:25,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:25,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:25,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:26,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:26,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:26,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:26,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:29,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:30,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:34,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:35,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:35,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:35,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:35,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:35,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:39,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:52:39,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:39,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:40,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:42,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:42,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:43,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:45,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:45,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:45,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:45,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:45,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:47,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:47,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:50,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:51,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:55,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:52:56,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:52:56,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:52:56,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:52:56,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:52:56,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:52:58,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:52:58,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:52:59,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:52:59,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:01,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:03,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:05,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:53:05,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:53:05,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:53:05,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:53:07,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:53:07,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:07,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:09,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:53:10,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:53:10,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:53:10,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:53:10,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:53:11,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:53:11,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:12,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:13,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:13,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:53:15,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:53:18,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:23,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:28,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:28,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:53:28,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:33,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:38,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:39,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:53:43,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:43,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:43,286 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:53:48,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:53,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:58,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:53:58,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:53:58,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:03,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:04,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,264 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_3ac6586c86738660d7b9.png with policy now +2023-07-12 03:54:04,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_7f02fda774e0cc0ec92a.png with policy now +2023-07-12 03:54:04,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_8544a6b986b0e7758e17.png with policy now +2023-07-12 03:54:04,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:54:04,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_920_4cbb72cb2a336b2f4f2b.png with policy now +2023-07-12 03:54:04,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:04,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:04,336 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:04,338 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:04,610 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7528qevp-media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 03:54:04,613 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9q5k60s-media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 03:54:04,684 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s3g25wpf-media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 03:54:04,822 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u99d50nw-media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 03:54:07,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:07,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 03:54:07,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 03:54:07,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 03:54:07,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 03:54:07,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:54:07,701 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/97d34b2f-media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 03:54:07,710 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jg6jp207-media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 03:54:07,721 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8yot8tdx-media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 03:54:07,731 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b2hed0lj-media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 03:54:09,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:54:09,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:11,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:13,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:13,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:14,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:14,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:15,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:19,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:20,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:23,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:23,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:23,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:23,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:23,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:25,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:27,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:27,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:28,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:28,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:31,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:31,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:33,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:33,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:33,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:33,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:35,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:35,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:36,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:39,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:54:39,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:42,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:42,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:42,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:42,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:42,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:43,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:43,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:44,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:44,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:54:47,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:47,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:51,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:53,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:54:53,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:53,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:54:53,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:54:53,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:54:55,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:54:55,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:58,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:54:59,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:54:59,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:54:59,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:03,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:03,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:03,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:03,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:03,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:03,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:07,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:07,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:09,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:55:09,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:11,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:12,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:12,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:12,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:12,850 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:14,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:15,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:55:15,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:15,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:15,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:19,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:20,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:23,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:24,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:24,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:24,067 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:24,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:26,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:27,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:27,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:30,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:55:30,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:31,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:31,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:35,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:35,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:35,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:35,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:35,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:35,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:37,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:39,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:55:39,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:55:39,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:55:39,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:55:39,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:55:39,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:55:39,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:42,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:43,655 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:55:45,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:55:45,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:55:48,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:53,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:55:58,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:00,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:00,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:04,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:09,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:56:10,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:15,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:15,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:15,076 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:20,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:25,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:30,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:30,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:30,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:33,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,710 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_a2e24d69369ee9545e6c.png with policy now +2023-07-12 03:56:33,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_98329b1eb9c39ec50100.png with policy now +2023-07-12 03:56:33,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_14811891692332578df2.png with policy now +2023-07-12 03:56:33,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:56:33,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_930_708543de31a4012af277.png with policy now +2023-07-12 03:56:33,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:56:33,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:56:33,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:56:33,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:56:34,030 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l05wg23a-media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 03:56:34,102 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ef5snuc4-media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 03:56:34,109 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4m30hi2l-media/images/validation_930_708543de31a4012af277.png +2023-07-12 03:56:34,206 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9rwgd1u2-media/images/validation_930_14811891692332578df2.png +2023-07-12 03:56:35,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:35,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:56:35,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_14811891692332578df2.png +2023-07-12 03:56:35,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 03:56:35,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 03:56:35,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_708543de31a4012af277.png +2023-07-12 03:56:35,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:56:36,211 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9oz98uum-media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 03:56:36,211 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2r9zkass-media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 03:56:36,580 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ssho1h6j-media/images/validation_930_14811891692332578df2.png +2023-07-12 03:56:36,631 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8w9fmdub-media/images/validation_930_708543de31a4012af277.png +2023-07-12 03:56:39,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:56:39,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:41,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:43,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:56:43,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:45,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:56:45,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:56:46,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:47,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:51,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:51,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:52,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:56:52,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:56:52,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:56:52,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:56:55,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:56:55,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:56:57,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:56:59,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:00,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:00,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:03,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:06,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 03:57:10,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:10,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:10,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:10,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:10,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:57:11,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:11,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:12,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:12,432 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:12,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:12,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:15,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:15,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:15,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:15,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:15,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:19,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:20,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:21,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:21,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:21,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:21,916 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:23,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:23,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:26,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:27,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:30,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:30,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:31,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:31,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:32,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:32,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:32,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:32,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:36,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:36,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:36,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:39,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:57:40,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:41,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:42,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:42,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:42,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:42,873 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:44,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:44,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:45,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:57:45,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:57:46,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:48,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:51,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:57:52,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:53,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:57:53,376 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:57:53,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:57:53,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:57:56,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:57:56,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:57:57,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:00,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:00,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:00,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:02,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:03,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:58:03,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:58:03,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:58:03,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:58:04,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:58:04,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:08,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:58:08,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:08,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:58:08,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:58:08,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:58:08,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:58:08,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:09,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:58:12,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:58:13,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:15,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:15,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:18,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:23,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:28,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:30,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:30,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:33,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:38,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:39,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:58:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:45,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:58:45,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:58:49,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:54,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:58:59,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:00,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:00,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:02,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_1aa1dd5671e7761dc7f9.png with policy now +2023-07-12 03:59:02,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_25baa72fa29a6b87a7e3.png with policy now +2023-07-12 03:59:02,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_55fb249dfc64d037ba51.png with policy now +2023-07-12 03:59:02,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 03:59:02,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_940_0f4ce489e03b7c6499a4.png with policy now +2023-07-12 03:59:02,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:02,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:02,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:02,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:03,006 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jltbj46f-media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 03:59:03,030 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cai0s1j8-media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 03:59:03,148 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f8ifli6d-media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 03:59:03,212 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/md76v5eb-media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 03:59:04,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:04,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 03:59:04,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 03:59:04,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 03:59:04,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 03:59:04,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 03:59:04,708 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wpb60rw1-media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 03:59:04,732 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4iqlfmn9-media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 03:59:04,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:08,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:09,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:59:10,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:10,461 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zol84uvh-media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 03:59:12,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:13,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:15,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:15,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:15,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:16,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:17,257 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ae26ohgk-media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 03:59:20,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:20,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:23,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:23,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:23,179 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:23,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:24,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:24,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:26,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:28,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:30,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:30,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:31,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:32,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:33,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:33,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:33,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:33,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:36,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:36,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:37,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:39,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 03:59:40,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:42,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:43,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:43,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:43,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:43,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:44,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:44,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:45,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 03:59:45,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 03:59:47,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:48,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:52,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:53,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 03:59:53,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 03:59:53,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 03:59:53,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 03:59:53,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 03:59:56,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 03:59:56,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 03:59:58,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:00,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:00,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:00,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:03,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:03,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:03,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:03,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:03,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:04,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:04,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:08,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:08,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:09,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:00:12,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:13,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:14,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:14,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:14,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:14,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:16,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:16,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:16,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:16,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:19,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:20,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:24,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:24,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:24,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:24,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:24,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:24,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:24,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:28,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:30,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:31,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:31,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:32,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:34,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:34,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:34,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:34,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:36,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:36,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:36,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:39,028 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:00:39,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:00:39,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:00:39,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:00:39,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:00:40,665 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:00:40,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:42,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:44,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:00:46,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:00:46,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:00:47,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:52,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:00:57,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:01,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:01,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:02,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:07,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:09,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:01:13,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:16,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:18,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:23,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:28,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:31,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:31,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:34,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_8e55f5be94ca16a527e8.png with policy now +2023-07-12 04:01:34,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:34,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_cedfd2e3a8d728719314.png with policy now +2023-07-12 04:01:34,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,078 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_b66230be0915df8b41e3.png with policy now +2023-07-12 04:01:34,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:01:34,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_950_32af62f9d0e790c02608.png with policy now +2023-07-12 04:01:34,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:01:34,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:01:34,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:01:34,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:01:34,386 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hpqqjty1-media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 04:01:34,413 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q0zoi9sc-media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 04:01:34,453 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6u2cy8p-media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 04:01:34,592 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qcl017ms-media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 04:01:36,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:01:36,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 04:01:36,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 04:01:36,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 04:01:36,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 04:01:36,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:01:37,236 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5si1vp9u-media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 04:01:37,307 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5x3cd3de-media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 04:01:37,312 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uvinis4v-media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 04:01:37,351 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jsiqxi9o-media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 04:01:39,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:01:40,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:40,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:44,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:01:44,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:45,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:46,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:01:46,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:01:48,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:51,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:52,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:01:53,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:01:53,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:01:53,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:01:53,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:01:56,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:01:56,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:01:56,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:00,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:01,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:01,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:01,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:03,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:03,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:03,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:03,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:04,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:04,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:06,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:08,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:09,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:02:12,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:12,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:14,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:14,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:14,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:14,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:16,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:16,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:16,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:17,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:20,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:23,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:24,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:24,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:24,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:24,300 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:25,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:25,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:28,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:29,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:31,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:31,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:33,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:33,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:34,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:34,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:34,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:34,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:37,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:37,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:39,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:02:39,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:41,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:44,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:44,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:44,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:44,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:44,286 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:45,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:45,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:46,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:02:46,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:02:49,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:49,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:53,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:02:55,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:02:55,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:02:55,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:02:55,360 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:02:55,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:02:57,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:02:57,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:01,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:01,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:02,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:02,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:05,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:03:05,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:03:05,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:03:05,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:03:05,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:03:05,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:07,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:09,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:03:09,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:09,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:03:09,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:03:09,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:03:09,705 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:03:12,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:13,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:03:13,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:03:17,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:17,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:18,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:23,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:28,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:32,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:32,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:34,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:39,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:03:40,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:45,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:47,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:03:47,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:03:50,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:03:55,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:00,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:02,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:02,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:04,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_a3b1bfcfc8aabde7212f.png with policy now +2023-07-12 04:04:04,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_5c7c719c7e9e5e608018.png with policy now +2023-07-12 04:04:04,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_fdf70bc0fef8dac9c1a0.png with policy now +2023-07-12 04:04:04,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:04:04,479 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_960_67790e930adf85923341.png with policy now +2023-07-12 04:04:04,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:04,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:04,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:04,510 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:04,758 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/10sycwqx-media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 04:04:04,780 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/68wt8gz6-media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 04:04:04,797 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/18x9tls0-media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 04:04:04,837 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4kul8u3v-media/images/validation_960_67790e930adf85923341.png +2023-07-12 04:04:05,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:05,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 04:04:05,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 04:04:05,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_67790e930adf85923341.png +2023-07-12 04:04:05,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 04:04:05,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:04:05,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:05,760 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5my200m3-media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 04:04:05,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/godsqtxa-media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 04:04:05,813 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zqw4jsfk-media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 04:04:06,018 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c0iy450d-media/images/validation_960_67790e930adf85923341.png +2023-07-12 04:04:09,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:04:09,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:10,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:13,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:14,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:16,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:17,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:17,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:17,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:21,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:21,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:24,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:24,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:24,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:24,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:25,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:25,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:27,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:29,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:32,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:32,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:33,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:33,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:34,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:34,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:34,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:34,600 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:37,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:37,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:39,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:39,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:04:41,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:44,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:44,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:44,996 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:44,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:44,998 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:45,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:45,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:47,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:04:47,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:04:49,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:49,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:53,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:04:55,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:04:55,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:04:55,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:04:55,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:04:55,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:04:57,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:04:57,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:00,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:01,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:02,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:02,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:05,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:05,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:05,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:05,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:05,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:05,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:09,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:05:09,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:09,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:11,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:13,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:16,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:16,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:16,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:16,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:16,529 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:17,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:17,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:17,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:17,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:21,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:21,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:25,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:26,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:26,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:26,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:26,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:26,539 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:29,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:29,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:32,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:33,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:33,228 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:33,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:36,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:36,187 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:36,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:36,189 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:37,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:37,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:38,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:39,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:05:40,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:05:40,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:05:40,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:05:40,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:05:41,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:05:41,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:43,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:45,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:05:48,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:05:48,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:05:49,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:54,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:05:59,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:03,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:03,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:05,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:09,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:06:11,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:16,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:18,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:18,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:21,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:26,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:31,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:33,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:33,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:35,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,917 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_4600d4c65eb0657bf15d.png with policy now +2023-07-12 04:06:35,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_133f183508d881d8f1f8.png with policy now +2023-07-12 04:06:35,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_959d30c344a53f583035.png with policy now +2023-07-12 04:06:35,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:06:35,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_970_783f685672728f7cab19.png with policy now +2023-07-12 04:06:35,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:06:35,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:06:35,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:06:35,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:06:36,237 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v69gl6yy-media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 04:06:36,254 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2np8c83-media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 04:06:36,264 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5u3dkcly-media/images/validation_970_783f685672728f7cab19.png +2023-07-12 04:06:36,420 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/afmhys1e-media/images/validation_970_959d30c344a53f583035.png +2023-07-12 04:06:36,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:37,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:06:37,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_783f685672728f7cab19.png +2023-07-12 04:06:37,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 04:06:37,896 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_959d30c344a53f583035.png +2023-07-12 04:06:37,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 04:06:37,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:06:38,405 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v0u6dxju-media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 04:06:38,419 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvzc122e-media/images/validation_970_959d30c344a53f583035.png +2023-07-12 04:06:38,455 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q6meisiw-media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 04:06:38,471 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kj4hwmmo-media/images/validation_970_783f685672728f7cab19.png +2023-07-12 04:06:39,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:06:41,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:42,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:45,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:46,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:06:48,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:48,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:06:48,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:06:49,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:53,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:06:53,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:55,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:06:55,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:06:55,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:06:55,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:06:57,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:06:57,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:06:58,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:01,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:03,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:03,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:03,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:05,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:06,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:06,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:06,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:06,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:09,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:07:09,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:09,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:09,969 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:13,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:14,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:15,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:15,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:15,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:15,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:17,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:17,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:18,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:18,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:19,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:21,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:25,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:25,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:25,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:25,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:25,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:26,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:26,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:30,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:30,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:33,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:33,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:34,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:35,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:35,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:35,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:35,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:35,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:38,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:38,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:39,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:07:41,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:42,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:46,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:46,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:46,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:46,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:46,528 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:46,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:48,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:07:48,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:07:50,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:50,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:51,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:54,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:07:56,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:07:56,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:07:56,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:07:56,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:07:56,745 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:07:58,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:07:58,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:02,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:02,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:03,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:03,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:06,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:06,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:08:06,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:08:06,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:08:06,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:08:08,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:09,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:08:10,187 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:08:10,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:11,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:08:11,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:08:11,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:08:11,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:08:13,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:14,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:08:14,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:08:18,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:18,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:19,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:24,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:29,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:33,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:33,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:34,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:39,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:08:40,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:45,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:48,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:08:48,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:08:50,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:08:55,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:00,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:03,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:03,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:05,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_5bce29b811f38d81b7a9.png with policy now +2023-07-12 04:09:05,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_4522b255d71fb495f911.png with policy now +2023-07-12 04:09:05,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,594 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_4a474756239679aa6533.png with policy now +2023-07-12 04:09:05,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:09:05,595 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_980_4e23c9690e64cda40444.png with policy now +2023-07-12 04:09:05,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:05,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:05,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:05,621 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:05,925 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/86vki8cg-media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 04:09:05,948 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9anp5i8n-media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 04:09:05,965 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8tdo0wcz-media/images/validation_980_4a474756239679aa6533.png +2023-07-12 04:09:06,080 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mz8pisi8-media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 04:09:06,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:06,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4a474756239679aa6533.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 04:09:06,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:09:06,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:06,748 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rz230x2s-media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 04:09:06,767 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/30a2fp2t-media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 04:09:06,838 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4uw6wve8-media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 04:09:07,621 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nsai1d5v-media/images/validation_980_4a474756239679aa6533.png +2023-07-12 04:09:09,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:09:12,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:14,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:15,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:17,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:18,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:18,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:18,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:22,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:22,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:24,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:24,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:24,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:24,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:26,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:26,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:28,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:30,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:33,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:34,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:34,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:34,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:34,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:34,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:34,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:34,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:38,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:38,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:39,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:09:39,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:42,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:44,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:44,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:44,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:44,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:44,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:46,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:49,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:09:49,210 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:09:49,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:50,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:54,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:09:54,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:09:54,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:09:54,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:09:54,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:09:54,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:09:55,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:09:58,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:00,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:02,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:04,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:04,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:04,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:04,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:04,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:04,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:05,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:06,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:06,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:09,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:10:10,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:11,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:14,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:15,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:15,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:15,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:15,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:17,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:18,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:18,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:19,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:19,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:22,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:22,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:25,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:25,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:25,560 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:25,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:26,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:26,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:28,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:30,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:33,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:34,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:35,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:35,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:35,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:35,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:35,982 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:35,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:38,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:38,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:39,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:10:39,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:40,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:10:40,394 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:10:40,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:10:40,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:10:42,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:10:42,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:10:45,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:50,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:10:50,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:10:50,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:10:55,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:00,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:05,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:05,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:06,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:09,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:11:12,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:17,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:20,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:20,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:22,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:27,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:32,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:34,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_e555318c04df0d898c35.png with policy now +2023-07-12 04:11:34,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,902 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_d4658b3a29548774d64f.png with policy now +2023-07-12 04:11:34,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_dd615cbf6d5a7a8a896d.png with policy now +2023-07-12 04:11:34,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:11:34,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_990_952fc214368e10b389b8.png with policy now +2023-07-12 04:11:34,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:11:34,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:11:34,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:11:34,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:11:35,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:11:35,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 04:11:35,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 04:11:35,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 04:11:35,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 04:11:35,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:11:35,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:35,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:38,077 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mgs1qv8y-media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 04:11:38,142 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gce1fiew-media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 04:11:38,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:39,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:39,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:11:39,257 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lb8d60xt-media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 04:11:39,919 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/trvo9gug-media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 04:11:39,996 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wr443tmh-media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 04:11:41,171 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aauszbr4-media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 04:11:41,552 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y6cvia2o-media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 04:11:41,921 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xww4uylq-media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 04:11:43,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:44,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:45,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:11:49,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:49,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:50,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:11:50,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:11:54,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:54,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:11:54,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:11:54,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:11:54,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:11:54,611 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:11:59,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:11:59,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:11:59,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:04,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:04,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:04,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:04,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:04,595 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:05,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:05,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:05,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:09,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:12:09,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:09,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:10,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:14,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:14,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:14,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:14,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:14,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:15,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:19,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:19,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:20,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:20,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:21,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:24,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:24,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:24,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:24,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:24,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:26,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:29,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:29,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:31,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:34,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:34,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:34,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:34,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:34,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:35,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:35,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:38,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:39,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:12:39,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:39,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:43,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:44,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:44,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:44,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:44,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:44,717 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:48,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:49,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:49,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:50,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:12:50,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:12:54,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:12:54,279 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:12:55,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:12:55,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:12:55,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:12:55,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:12:59,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:12:59,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:00,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:04,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:05,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:05,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:13:05,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:13:05,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:13:05,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:13:05,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:05,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:09,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:13:09,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:13:09,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:09,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:13:09,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:13:09,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:13:09,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:13:10,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:14,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:13:14,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:13:15,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:20,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:20,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:20,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:25,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:30,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:35,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:35,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:35,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:39,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:13:41,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:46,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:50,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:13:50,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:13:51,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:13:56,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:01,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:04,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,116 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_1a96794a04b7af41b34e.png with policy now +2023-07-12 04:14:04,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_e0021fa68cabb943f49d.png with policy now +2023-07-12 04:14:04,158 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_87434c7eeda4190bf8d8.png with policy now +2023-07-12 04:14:04,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:14:04,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1000_bd2b2cab96e55845d6e1.png with policy now +2023-07-12 04:14:04,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:04,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:04,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:04,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:04,452 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j49g4fzt-media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 04:14:04,490 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uhorvb1d-media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 04:14:04,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:04,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 04:14:04,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 04:14:04,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 04:14:04,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 04:14:04,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:14:04,497 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9wn9d9fa-media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 04:14:04,609 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6h89ijj9-media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 04:14:04,845 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ijb7p9dn-media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 04:14:04,906 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i36nkybh-media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 04:14:04,984 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/txr7czh6-media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 04:14:05,060 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2dkoe1rx-media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 04:14:06,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:06,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:07,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:09,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:14:09,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:12,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:13,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:14,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:18,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:19,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:21,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:21,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:23,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:23,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:23,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:23,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:23,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:24,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:24,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:29,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:29,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:34,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:34,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:34,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:34,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:34,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:34,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:35,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:36,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:36,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:39,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:14:39,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:40,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:43,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:43,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:43,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:43,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:44,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:44,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:45,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:49,589 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:50,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:51,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:14:51,505 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:14:54,599 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:14:54,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:14:54,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:14:54,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:14:54,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:14:56,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:14:59,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:14:59,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:02,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:04,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:04,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:04,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:04,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:04,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:07,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:07,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:07,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:09,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:15:09,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:09,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:13,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:14,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:15,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:15,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:15,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:15,052 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:19,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:19,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:19,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:22,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:22,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:24,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:25,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:25,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:25,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:25,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:25,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:29,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:29,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:30,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:34,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:35,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:35,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:35,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:35,090 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:36,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:37,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:37,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:39,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:15:39,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:15:39,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:15:39,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:15:39,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:15:39,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:15:39,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:41,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:44,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:15:46,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:51,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:15:52,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:15:52,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:15:57,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:02,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:07,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:07,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:07,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:09,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:16:13,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:18,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:22,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:22,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:23,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:28,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:33,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:34,224 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_813871e148b66856b8a5.png with policy now +2023-07-12 04:16:34,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_f52e87f356214ba43ded.png with policy now +2023-07-12 04:16:34,266 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_8213e8a29e4df4a0c573.png with policy now +2023-07-12 04:16:34,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:16:34,267 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1010_1fac8c596692012a9a71.png with policy now +2023-07-12 04:16:34,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:16:34,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:16:34,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:16:34,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:16:34,547 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tqa77qvj-media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 04:16:34,576 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jm7m0jdo-media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 04:16:34,588 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgy46oib-media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 04:16:34,594 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/661cromi-media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 04:16:34,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:16:34,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 04:16:34,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 04:16:34,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 04:16:34,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 04:16:34,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:16:35,376 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1ioaof55-media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 04:16:35,384 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8gbbcprc-media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 04:16:35,400 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2ls9nzwc-media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 04:16:35,624 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ndoiutho-media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 04:16:37,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:37,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:38,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:39,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:16:39,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:43,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:45,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:45,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:16:48,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:50,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:52,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:16:52,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:16:54,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:16:54,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:16:54,511 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:16:54,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:16:54,514 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:16:55,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:16:55,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:16:59,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:00,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:04,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:04,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:04,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:04,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:04,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:05,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:05,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:07,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:07,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:09,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:17:10,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:10,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:15,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:15,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:15,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:15,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:15,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:16,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:20,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:20,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:21,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:22,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:22,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:25,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:25,070 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:25,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:25,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:25,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:25,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:27,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:30,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:32,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:34,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:34,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:34,876 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:34,878 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:35,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:35,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:37,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:37,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:37,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:39,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:17:40,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:42,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:45,222 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:45,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:45,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:45,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:45,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:48,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:50,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:17:50,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:53,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:17:53,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:17:53,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:17:55,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:17:55,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:17:55,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:17:55,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:17:55,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:17:59,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:00,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:18:00,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:05,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:05,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:06,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:18:06,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:18:06,132 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:18:06,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:18:08,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:08,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:09,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:18:10,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:18:10,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:10,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:18:10,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:10,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:18:10,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:18:10,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:18:15,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:18:15,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:18:16,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:21,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:23,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:23,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:27,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:32,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:37,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:38,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:38,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:39,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:18:43,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:48,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:53,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:18:53,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:18:53,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:18:58,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:03,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:05,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_f7760d16dbb61cdadc4b.png with policy now +2023-07-12 04:19:05,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 04:19:05,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 04:19:05,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 04:19:05,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 04:19:05,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:19:05,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_bc519090f4c42a857cf5.png with policy now +2023-07-12 04:19:05,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_d02fa8190f5853b0b648.png with policy now +2023-07-12 04:19:05,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:19:05,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1020_f59fa5400ab07ea1eeae.png with policy now +2023-07-12 04:19:05,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:05,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:05,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:05,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:05,730 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mc8liz44-media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 04:19:05,771 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gyqkvym0-media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 04:19:05,796 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/14krwzs0-media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 04:19:05,806 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aj3w221o-media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 04:19:06,143 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9zfp7o13-media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 04:19:06,164 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3jz6whro-media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 04:19:06,310 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/le982l75-media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 04:19:06,588 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jw64ai70-media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 04:19:08,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:08,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:08,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:09,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:19:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:10,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:14,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:15,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:15,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:19,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:20,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:23,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:24,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:24,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:24,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:24,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:25,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:25,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:25,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:30,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:30,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:33,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:33,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:33,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:33,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:35,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:35,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:36,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:38,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:38,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:39,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:19:40,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:41,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:43,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:43,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:43,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:43,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:45,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:45,537 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:46,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:50,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:52,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:19:53,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:19:53,296 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:19:53,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:19:53,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:19:53,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:19:53,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:19:55,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:19:55,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:19:58,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:00,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:03,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:04,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:04,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:04,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:04,004 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:05,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:05,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:08,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:08,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:08,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:09,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:20:10,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:14,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:15,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:15,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:15,143 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:15,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:15,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:15,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:19,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:20,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:23,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:23,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:24,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:25,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:25,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:25,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:25,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:25,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:25,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:30,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:35,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:35,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:35,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:35,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:35,627 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:35,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:35,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:38,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:38,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:39,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:20:40,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:40,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:20:40,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:20:40,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:20:40,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:20:40,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:20:40,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:45,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:20:46,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:51,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:20:53,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:20:53,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:20:56,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:01,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:06,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:09,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:09,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:09,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:21:12,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:17,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:22,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:24,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:24,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:27,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:32,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:35,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_bfd7171975011562507f.png with policy now +2023-07-12 04:21:35,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,053 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_fd568449fa00bd93527b.png with policy now +2023-07-12 04:21:35,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_a95db61e150a9ffc4641.png with policy now +2023-07-12 04:21:35,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:21:35,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1030_3856c65003790d9a56ab.png with policy now +2023-07-12 04:21:35,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:21:35,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:21:35,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:21:35,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:21:35,377 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nfiglqqz-media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 04:21:35,380 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pb5a5dau-media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 04:21:35,416 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ku0qug25-media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 04:21:35,497 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c0k58urc-media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 04:21:35,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:21:35,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 04:21:35,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 04:21:35,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 04:21:35,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 04:21:35,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:21:36,261 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4j4eyfbg-media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 04:21:36,302 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i2cajqmg-media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 04:21:36,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e9a25pap-media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 04:21:36,353 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/08nsv9z0-media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 04:21:38,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:39,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:21:39,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:39,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:40,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:43,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:45,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:21:45,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:49,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:50,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:54,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:21:55,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:21:55,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:21:55,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:21:55,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:21:55,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:21:55,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:21:55,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:21:55,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:21:59,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:00,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:05,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:05,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:05,883 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:05,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:05,885 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:05,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:05,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:09,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:22:10,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:10,716 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:10,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:10,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:16,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:16,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:16,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:16,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:16,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:16,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:21,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:21,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:21,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:25,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:25,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:25,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:25,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:25,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:25,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:26,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:26,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:26,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:31,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:32,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:35,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:35,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:35,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:35,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:36,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:36,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:37,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:39,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:22:41,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:41,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:41,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:43,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:46,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:46,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:46,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:46,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:46,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:48,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:51,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:22:51,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:54,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:22:56,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:22:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:22:56,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:22:57,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:22:57,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:22:57,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:22:57,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:22:59,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:01,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:23:01,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:05,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:06,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:07,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:23:07,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:23:07,429 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:23:07,431 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:23:09,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:23:10,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:11,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:23:11,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:11,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:11,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:23:12,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:23:12,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:23:12,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:23:12,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:23:16,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:16,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:23:16,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:23:21,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:26,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:26,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:26,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:23:31,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:36,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:39,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:23:41,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:41,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:41,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:23:46,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:51,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:56,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:23:56,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:23:56,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:01,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:06,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_15446d27b696fee5751e.png with policy now +2023-07-12 04:24:06,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_9a1046e9395d322b203b.png with policy now +2023-07-12 04:24:06,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_e6fd436066e140c8ca6d.png with policy now +2023-07-12 04:24:06,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:24:06,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1040_341a0c23b11ea49f2577.png with policy now +2023-07-12 04:24:06,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:06,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:06,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:06,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:07,139 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z056jppi-media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 04:24:07,175 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27tfjrpu-media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 04:24:07,207 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b7wmp72a-media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 04:24:07,216 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/822hl4m2-media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 04:24:07,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:09,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:24:11,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:11,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:11,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 04:24:11,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 04:24:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 04:24:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 04:24:11,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:24:11,765 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xrc6jqm8-media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 04:24:11,766 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/josf8kte-media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 04:24:11,830 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mxslmvcm-media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 04:24:11,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:11,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:11,969 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9z2664lz-media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 04:24:12,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:16,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:16,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:18,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:21,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:24,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:25,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:25,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:25,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:25,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:26,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:26,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:27,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:27,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:29,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:31,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:35,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:35,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:35,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:35,042 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:35,044 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:36,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:36,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:39,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:24:40,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:41,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:42,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:42,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:24:44,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:44,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:44,995 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:44,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:45,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:51,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:51,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:55,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:24:55,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:24:55,320 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:24:55,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:24:56,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:24:56,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:24:57,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:24:57,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:24:57,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:01,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:02,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:05,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:05,772 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:05,772 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:05,774 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:06,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:06,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:07,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:09,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:25:11,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:12,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:12,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:13,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:16,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:16,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:16,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:16,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:16,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:18,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:21,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:21,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:24,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:26,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:27,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:27,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:28,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:28,070 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:28,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:28,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:30,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:31,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:31,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:35,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:36,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:38,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:38,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:38,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:38,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:39,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:25:41,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:41,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:41,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:42,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:42,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:25:43,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:25:43,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:25:43,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:25:43,477 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:25:46,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:25:46,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:25:47,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:52,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:57,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:25:57,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:25:57,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:02,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:07,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:09,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:26:12,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:12,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:13,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:18,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:23,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:28,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:28,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:29,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:34,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:37,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_55de70da5261f9290e40.png with policy now +2023-07-12 04:26:37,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,622 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_f35c05f1d08c6308ff14.png with policy now +2023-07-12 04:26:37,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_9bfabce8c467677fe54b.png with policy now +2023-07-12 04:26:37,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:26:37,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1050_1daa5fdb77023a407299.png with policy now +2023-07-12 04:26:37,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:26:37,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:26:37,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:26:37,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:26:37,895 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gxtuxo00-media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 04:26:37,950 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kw17xklj-media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 04:26:37,951 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iqki9t7e-media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 04:26:37,954 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f32n8w7i-media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 04:26:39,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:26:39,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:41,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:26:41,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 04:26:41,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 04:26:41,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 04:26:41,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 04:26:41,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:26:42,225 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wo4kjlhy-media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 04:26:42,246 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sxqh33u9-media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 04:26:42,260 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9uodbpe8-media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 04:26:42,275 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ylp4ehu4-media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 04:26:43,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:43,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:26:44,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:46,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:26:47,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:26:50,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:51,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:26:55,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:26:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:26:57,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:26:57,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:26:57,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:26:57,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:26:58,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:26:58,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:00,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:01,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:01,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:06,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:06,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:07,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:07,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:07,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:07,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:09,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:27:11,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:11,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:11,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:13,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:13,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:16,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:16,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:18,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:18,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:18,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:18,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:21,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:21,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:21,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:26,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:27,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:28,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:28,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:28,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:28,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:28,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:28,948 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:31,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:31,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:32,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:36,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:38,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:38,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:38,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:38,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:38,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:39,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:27:42,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:42,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:43,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:43,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:43,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:47,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:49,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:49,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:49,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:49,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:49,254 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:27:52,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:27:52,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:55,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:27:57,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:27:58,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:27:58,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:27:59,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:27:59,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:27:59,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:27:59,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:28:00,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:02,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:28:02,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:05,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:07,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:09,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:28:10,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:28:10,109 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:28:10,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:28:10,111 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:28:11,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:12,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:28:12,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:14,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:14,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:28:14,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:28:14,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:28:14,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:28:14,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:28:16,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:17,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:28:17,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:28:21,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:26,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:29,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:29,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:28:32,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:37,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:39,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:28:43,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:44,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:44,353 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:28:48,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:53,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:58,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:28:59,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:28:59,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:04,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:09,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:29:09,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,201 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_197022e2780b05209f12.png with policy now +2023-07-12 04:29:09,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_06ba5b400f585cfe22b9.png with policy now +2023-07-12 04:29:09,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_7be7ad3a7a700e0a150c.png with policy now +2023-07-12 04:29:09,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:29:09,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1060_23fba533f1fd02e857f8.png with policy now +2023-07-12 04:29:09,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:09,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:09,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:09,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:09,517 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/885zd2w6-media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 04:29:09,537 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0sahtjjz-media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 04:29:09,545 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mq7vmlkt-media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 04:29:09,554 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/exl5eo5i-media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 04:29:10,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:12,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:12,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 04:29:12,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 04:29:12,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 04:29:12,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 04:29:12,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:29:12,723 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dbpc8hbn-media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 04:29:12,747 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hi4lozlj-media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 04:29:12,898 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0nclduzr-media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 04:29:12,908 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hn4ilugs-media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 04:29:14,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:29:14,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:15,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:17,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:19,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:21,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:22,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:26,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:27,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:27,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:27,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:27,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:27,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:29,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:29:29,579 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:32,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:32,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:32,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:37,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:37,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:37,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:37,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:37,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:37,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:37,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:39,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:29:42,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:43,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:45,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:29:45,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:29:47,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:48,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:48,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:48,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:48,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:29:49,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:52,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:29:52,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:54,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:29:57,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:29:58,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:29:58,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:29:58,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:29:58,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:00,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:00,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:00,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:02,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:02,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:05,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:07,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:09,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:09,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:09,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:09,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:09,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:30:11,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:12,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:12,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:15,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:15,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:16,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:17,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:19,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:19,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:19,286 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:19,288 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:21,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:22,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:22,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:27,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:27,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:28,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:28,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:28,971 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:28,973 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:30,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:30,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:32,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:32,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:33,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:37,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:38,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:38,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:38,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:38,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:38,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:39,099 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:30:42,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:42,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:43,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:30:43,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:30:43,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:30:43,568 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:30:43,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:45,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:30:45,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:30:47,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:30:47,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:30:49,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:54,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:30:59,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:00,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:00,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:05,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:09,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:31:11,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:15,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:15,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:16,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:21,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:26,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:30,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:30,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:31,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:36,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:37,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:37,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_32f9f6540dbdc7c48f81.png with policy now +2023-07-12 04:31:38,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:38,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_68b196a64c6e776ccd9d.png with policy now +2023-07-12 04:31:38,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:38,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_bd0d7850e520c4cb0100.png with policy now +2023-07-12 04:31:38,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:31:38,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1070_a070795b58d5c596307b.png with policy now +2023-07-12 04:31:38,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:31:38,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:31:38,054 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:31:38,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:31:38,271 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ql5afo80-media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 04:31:38,333 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zeijnye-media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 04:31:38,339 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z5q9i5zz-media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 04:31:38,342 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mths02qa-media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 04:31:39,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:31:42,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:42,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:31:42,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:42,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 04:31:42,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 04:31:42,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 04:31:42,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 04:31:42,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:31:43,220 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4eaqe12-media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 04:31:43,241 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ow2maca-media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 04:31:43,259 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m7nllooj-media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 04:31:43,287 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/klq2j92z-media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 04:31:45,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:31:45,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:31:47,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:47,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:47,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:31:52,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:53,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:31:56,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:31:56,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:31:56,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:31:56,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:31:57,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:31:57,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:31:58,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:01,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:01,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:02,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:04,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:07,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:07,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:07,460 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:07,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:07,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:07,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:09,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:32:09,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:12,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:14,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:16,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:16,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:17,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:17,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:17,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:17,248 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:17,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:17,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:20,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:22,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:25,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:27,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:28,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:28,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:28,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:28,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:30,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:32,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:32,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:32,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:32,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:36,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:37,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:37,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:37,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:37,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:37,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:37,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:39,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:32:41,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:43,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:46,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:47,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:32:47,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:32:48,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:48,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:48,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:48,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:48,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:32:51,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:53,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:32:53,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:57,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:32:58,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:32:59,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:32:59,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:32:59,125 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:32:59,127 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:33:02,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:03,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:33:03,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:03,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:03,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:08,117 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:08,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:09,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:33:09,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:33:09,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:33:09,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:33:09,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:33:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:33:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:14,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:14,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:33:14,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:33:14,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:33:14,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:33:18,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:33:18,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:33:18,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:18,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:19,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:24,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:29,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:33,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:33,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:34,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:39,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:33:40,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:45,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:48,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:33:48,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:33:50,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:33:55,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:00,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:03,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:03,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:05,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:09,109 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:34:09,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_1f6acd9fcf45ba3ea19f.png with policy now +2023-07-12 04:34:09,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,230 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_4817a2b823e0b176afd7.png with policy now +2023-07-12 04:34:09,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_66da8376f918e5f31cac.png with policy now +2023-07-12 04:34:09,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:34:09,231 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1080_6b2cf11ab85982d34290.png with policy now +2023-07-12 04:34:09,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:09,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:09,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:09,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:09,522 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ouzuahv-media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 04:34:09,571 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mhn9zc25-media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 04:34:09,588 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/40m8p3y0-media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 04:34:09,597 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cv0geh5t-media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 04:34:11,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:13,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:13,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:13,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 04:34:13,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 04:34:13,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 04:34:13,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 04:34:13,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:34:13,698 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zy4s9pss-media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 04:34:13,709 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ppaltm0f-media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 04:34:13,819 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sez5n7i9-media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 04:34:13,864 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fx4cja0g-media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 04:34:16,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:18,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:18,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:18,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:19,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:21,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:23,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:27,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:28,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:28,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:28,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:28,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:28,660 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:32,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:33,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:33,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:34,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:34,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:37,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:38,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:38,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:38,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:38,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:38,971 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:39,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:34:43,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:43,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:43,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:48,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:48,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:49,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:34:49,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:34:49,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:49,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:49,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:49,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:34:53,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:34:53,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:53,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:58,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:34:58,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:34:59,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:34:59,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:34:59,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:34:59,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:03,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:03,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:04,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:04,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:04,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:08,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:09,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:35:10,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:10,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:10,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:10,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:10,738 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:13,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:13,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:15,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:18,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:19,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:19,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:20,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:20,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:20,207 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:20,209 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:21,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:23,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:23,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:26,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:28,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:30,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:30,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:30,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:30,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:31,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:33,582 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:33,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:35,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:35,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:37,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:38,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:39,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:35:40,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:40,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:40,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:40,626 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:42,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:43,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:43,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:45,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:35:45,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:35:45,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:35:45,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:35:48,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:35:48,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:35:48,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:50,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:35:50,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:35:54,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:35:59,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:04,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:05,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:05,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:09,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:36:09,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:14,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:19,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:20,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:20,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:25,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:30,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:35,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:35,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:35,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:39,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:36:39,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_b8740b576006b9ee3546.png with policy now +2023-07-12 04:36:39,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,361 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_76a3173208a51ee8c312.png with policy now +2023-07-12 04:36:39,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_057775edd61cac3840c0.png with policy now +2023-07-12 04:36:39,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:36:39,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1090_b585e13abb070740f049.png with policy now +2023-07-12 04:36:39,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:36:39,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:36:39,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:36:39,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:36:39,670 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6b7kof9s-media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 04:36:39,673 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lpakgm1u-media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 04:36:39,676 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e41tyzyk-media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 04:36:39,752 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/scm0l7e1-media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 04:36:40,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:43,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 04:36:43,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 04:36:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 04:36:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:36:44,134 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o86u4jgp-media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 04:36:44,153 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1s7pqxud-media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 04:36:44,172 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3smzremv-media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 04:36:44,173 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r1umyyrj-media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 04:36:46,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:48,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:49,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:36:50,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:36:50,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:36:51,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:53,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:56,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:36:58,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:36:59,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:36:59,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:36:59,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:36:59,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:02,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:03,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:03,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:05,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:05,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:08,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:08,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:09,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:37:09,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:09,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:09,541 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:09,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:13,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:13,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:13,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:18,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:18,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:20,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:20,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:20,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:20,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:20,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:20,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:23,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:23,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:24,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:28,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:30,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:30,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:30,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:30,892 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:30,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:33,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:33,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:35,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:35,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:35,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:38,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:39,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:37:40,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:40,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:40,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:40,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:40,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:44,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:44,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:46,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:49,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:50,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:37:50,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:37:50,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:37:50,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:37:50,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:37:50,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:37:51,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:54,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:37:54,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:37:57,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:37:59,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:01,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:38:01,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:38:01,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:38:01,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:38:02,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:04,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:38:04,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:05,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:05,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:08,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:09,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:09,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:38:11,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:38:11,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:38:11,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:38:11,728 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:38:13,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:14,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:38:14,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:16,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:38:16,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:38:16,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:38:16,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:38:19,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:38:19,101 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:19,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:20,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:20,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:24,112 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:38:24,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:29,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:34,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:35,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:35,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:39,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:38:40,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:45,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:50,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:38:50,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:38:50,593 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:38:55,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:00,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:05,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:05,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:05,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:09,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:39:11,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:11,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_b9ea83bc203cb55d812b.png with policy now +2023-07-12 04:39:11,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_ec943a3b531d5c037829.png with policy now +2023-07-12 04:39:11,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_61651029309c609a7521.png with policy now +2023-07-12 04:39:11,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:39:11,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1100_1ad4a61cbb8909fcb78e.png with policy now +2023-07-12 04:39:11,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:11,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:11,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:11,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:11,596 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l0fg1gpf-media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 04:39:11,638 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p2g16zrg-media/images/validation_1100_61651029309c609a7521.png +2023-07-12 04:39:11,653 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tax1kv77-media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 04:39:11,713 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yisr6r7c-media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 04:39:14,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:14,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 04:39:14,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 04:39:14,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 04:39:14,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_61651029309c609a7521.png +2023-07-12 04:39:14,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:39:14,694 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/470ic08f-media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 04:39:14,703 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qp1w3fcp-media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 04:39:14,707 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6m3i5jxk-media/images/validation_1100_61651029309c609a7521.png +2023-07-12 04:39:14,965 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zbeb97i7-media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 04:39:16,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:19,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:20,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:20,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:21,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:21,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:24,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:26,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:29,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:30,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:30,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:30,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:30,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:31,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:34,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:34,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:35,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:35,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:36,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:39,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:39:39,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:40,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:40,262 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:40,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:40,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:42,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:44,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:44,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:47,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:49,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:50,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:39:50,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:39:51,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:39:51,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:39:51,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:39:51,131 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:39:53,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:54,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:39:54,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:39:58,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:39:59,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:01,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:01,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:01,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:01,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:03,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:04,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:04,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:05,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:05,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:09,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:09,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:40:09,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:12,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:12,094 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:12,094 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:12,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:14,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:14,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:14,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:19,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:19,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:21,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:21,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:22,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:22,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:22,334 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:22,336 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:24,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:24,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:24,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:29,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:30,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:32,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:32,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:32,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:32,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:34,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:34,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:36,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:36,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:36,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:39,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:40:39,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:41,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:42,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:42,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:42,110 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:42,112 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:44,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:44,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:46,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:46,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:40:46,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:40:46,938 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:40:46,940 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:40:49,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:40:49,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:40:51,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:40:51,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:40:51,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:40:56,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:01,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:06,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:06,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:06,749 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:09,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:41:12,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:17,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:21,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:21,749 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:22,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:27,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:32,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:36,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:36,776 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:37,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:39,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:41:41,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_28d2a849740383813cb6.png with policy now +2023-07-12 04:41:41,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,286 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_b889d46b029a022ecfc9.png with policy now +2023-07-12 04:41:41,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_54b9949ee470b53f0543.png with policy now +2023-07-12 04:41:41,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:41:41,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1110_eb513f39f1c260344506.png with policy now +2023-07-12 04:41:41,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:41:41,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:41:41,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:41:41,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:41:41,647 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/onq0sv4z-media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 04:41:41,660 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9291c8h0-media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 04:41:41,668 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dzep3ruk-media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 04:41:41,717 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zknqrqt3-media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 04:41:43,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:44,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:41:44,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 04:41:44,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 04:41:44,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 04:41:44,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 04:41:44,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:41:45,178 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9m3xkejl-media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 04:41:45,192 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oov39fny-media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 04:41:45,193 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y0f2gfbf-media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 04:41:45,307 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyfczars-media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 04:41:48,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:49,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:41:51,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:41:51,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:41:51,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:41:54,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:54,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:41:59,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:41:59,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:00,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:00,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:00,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:00,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:04,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:04,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:04,870 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:07,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:07,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:09,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:42:09,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:09,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:09,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:09,776 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:09,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:09,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:09,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:14,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:14,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:19,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:19,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:19,714 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:19,716 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:19,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:19,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:20,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:22,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:22,147 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:24,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:25,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:29,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:30,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:30,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:30,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:30,110 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:31,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:34,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:34,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:36,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:38,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:38,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:39,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:42:39,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:41,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:41,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:41,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:41,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:41,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:45,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:45,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:47,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:50,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:52,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:42:52,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:42:52,315 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:42:52,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:42:52,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:42:53,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:42:53,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:42:55,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:42:55,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:42:57,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:00,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:02,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:43:02,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:02,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:43:02,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:43:02,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:43:05,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:43:05,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:08,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:08,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:08,155 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:09,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:43:10,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:12,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:43:12,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:43:12,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:43:12,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:43:13,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:15,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:43:15,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:17,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:43:17,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:43:17,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:43:17,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:43:19,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:20,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:43:20,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:43:23,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:23,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:24,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:29,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:34,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:38,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:38,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:39,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:43:39,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:44,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:49,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:43:53,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:43:53,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:43:55,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:00,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:05,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:08,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:08,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:09,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:44:11,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:11,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_3bb3ea14d134cd57deb1.png with policy now +2023-07-12 04:44:11,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,958 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_2c6f306607bc7b9bfd92.png with policy now +2023-07-12 04:44:11,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_8c2839a1d3140c3a7dfe.png with policy now +2023-07-12 04:44:11,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:44:11,979 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1120_fd6b36b89f1e7fd2e78a.png with policy now +2023-07-12 04:44:12,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:12,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:12,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:12,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:12,271 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ic8e69s9-media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 04:44:12,282 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b8m4ctxf-media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 04:44:12,283 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u8pfiyuv-media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 04:44:12,367 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/toi8yswl-media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 04:44:15,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 04:44:15,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 04:44:15,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:44:15,664 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/knin30hy-media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 04:44:15,678 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4i0zma8-media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 04:44:15,681 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4k84dvv1-media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 04:44:15,701 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0v1899xp-media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 04:44:16,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:20,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:21,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:22,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:23,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:23,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:25,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:28,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:30,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:31,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:31,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:31,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:31,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:33,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:35,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:35,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:38,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:38,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:38,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:39,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:44:40,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:40,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:40,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:40,935 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:40,937 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:43,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:45,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:45,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:49,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:50,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:50,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:44:50,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:44:50,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:44:50,687 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:44:53,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:44:53,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:44:54,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:44:55,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:44:55,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:44:59,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:00,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:01,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:01,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:01,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:01,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:05,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:05,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:05,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:09,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:09,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:09,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:45:10,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:10,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:12,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:12,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:12,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:12,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:15,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:15,549 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:16,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:20,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:21,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:23,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:23,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:23,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:23,133 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:24,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:24,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:25,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:25,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:27,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:30,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:32,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:33,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:33,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:33,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:33,184 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:35,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:35,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:37,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:39,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:45:39,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:39,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:40,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:42,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:42,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:42,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:42,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:42,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:45,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:45,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:48,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:45:48,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:48,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:45:48,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:45:48,071 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:45:50,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:45:50,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:45:53,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:45:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:45:54,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:45:59,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:04,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:09,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:09,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:46:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:09,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:14,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:19,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:24,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:24,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:25,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:30,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:35,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:39,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:46:39,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:39,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:40,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:42,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_93e81f1f307564b83df8.png with policy now +2023-07-12 04:46:42,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_331aea85c432690f9c0e.png with policy now +2023-07-12 04:46:42,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_44a8006c0888f7a9b714.png with policy now +2023-07-12 04:46:42,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:46:42,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1130_50f80b0ec1f9f7e17923.png with policy now +2023-07-12 04:46:42,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:46:42,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:46:42,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:46:42,549 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:46:42,823 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b61qzhuz-media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 04:46:42,847 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c1bg5fiv-media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 04:46:42,848 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icsen1n5-media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 04:46:42,881 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w0zhxpwj-media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 04:46:45,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:46:45,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 04:46:45,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 04:46:45,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 04:46:45,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 04:46:45,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:46:46,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:46,194 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x5o8fjqn-media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 04:46:46,201 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3mrskffr-media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 04:46:46,243 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83c6mbmu-media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 04:46:46,255 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d359d2so-media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 04:46:50,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:46:51,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:46:51,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:46:54,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:46:54,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:46:55,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:46:56,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:00,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:01,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:01,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:01,898 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:01,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:01,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:05,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:05,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:07,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:09,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:47:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:09,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:47:10,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:11,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:11,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:11,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:11,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:13,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:15,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:15,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:18,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:20,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:21,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:21,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:21,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:21,080 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:24,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:25,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:25,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:47:25,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:25,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:29,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:30,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:31,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:31,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:31,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:31,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:35,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:35,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:35,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:39,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:47:40,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:40,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:41,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:41,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:47:42,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:42,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:42,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:42,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:45,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:46,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:46,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:50,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:51,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:52,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:47:52,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:47:52,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:47:52,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:47:55,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:47:56,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:47:56,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:47:57,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:47:57,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:01,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:01,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:02,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:48:02,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:48:02,973 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:48:02,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:48:06,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:48:06,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:07,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:09,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:48:11,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:12,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:12,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:12,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:13,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:48:13,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:48:13,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:48:13,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:48:16,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:48:16,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:18,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:18,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:48:18,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:48:18,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:48:18,360 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:48:21,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:48:21,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:48:23,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:27,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:27,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:29,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:34,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:39,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:48:40,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:42,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:42,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:48:45,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:50,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:55,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:48:57,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:48:57,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:00,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:05,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:09,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:49:11,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:12,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,618 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_1961fdba2d4312886e8c.png with policy now +2023-07-12 04:49:12,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_49f1b6dda25d69bd1eec.png with policy now +2023-07-12 04:49:12,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_a158acb6a30a6a4fcc09.png with policy now +2023-07-12 04:49:12,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:49:12,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1140_469091e483586cec37b1.png with policy now +2023-07-12 04:49:12,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:12,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:12,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:12,704 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:12,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:12,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:12,943 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cname3sw-media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 04:49:12,969 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pz4i38rj-media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 04:49:12,997 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1va1i1yw-media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 04:49:13,010 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ophx3tba-media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 04:49:16,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:16,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 04:49:16,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 04:49:16,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 04:49:16,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 04:49:16,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:49:16,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:16,778 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x7quqeo5-media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 04:49:16,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qs2drvpr-media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 04:49:16,780 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3njsz0fo-media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 04:49:16,781 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1ly5ch2-media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 04:49:21,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:22,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:22,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:26,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:27,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:27,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:27,810 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:31,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:32,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:32,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:32,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:32,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:33,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:36,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:36,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:38,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:39,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:49:41,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:41,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:41,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:41,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:41,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:43,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:43,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:49:44,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:46,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:46,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:49,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:51,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:52,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:49:52,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:49:52,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:49:52,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:49:54,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:49:56,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:49:56,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:49:59,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:49:59,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:00,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:01,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:02,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:02,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:02,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:02,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:06,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:06,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:06,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:09,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:50:11,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:12,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:12,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:12,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:12,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:12,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:14,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:50:14,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:16,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:16,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:17,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:21,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:22,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:22,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:22,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:22,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:22,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:26,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:26,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:28,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:29,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:50:29,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:31,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:32,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:32,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:32,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:32,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:33,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:36,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:36,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:39,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:50:39,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:41,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:43,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:43,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:43,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:43,270 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:44,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:45,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:50:45,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:50:46,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:46,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:48,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:50:48,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:50:48,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:50:48,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:50:49,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:51,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:50:51,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:50:54,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:50:59,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:00,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:00,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:05,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:09,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:51:11,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:15,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:15,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:16,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:21,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:26,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:30,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:30,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:31,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:36,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:39,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:51:42,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:42,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_9d89ef355de68c0d9ddd.png with policy now +2023-07-12 04:51:42,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_bc32f73444fde5093291.png with policy now +2023-07-12 04:51:42,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_c9a41cef5442cfbcf575.png with policy now +2023-07-12 04:51:42,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:51:42,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1150_4646158f3cc2ef18d62e.png with policy now +2023-07-12 04:51:42,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:51:42,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:51:42,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:51:42,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:51:43,193 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n9pkzkf5-media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 04:51:43,247 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/szoeplkq-media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 04:51:43,257 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hnczw4cy-media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 04:51:43,266 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjkbsynj-media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 04:51:45,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:51:45,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:51:46,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:51:46,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:51:46,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 04:51:46,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 04:51:46,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 04:51:46,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 04:51:46,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:51:47,207 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q0pl6f7d-media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 04:51:47,229 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cnpn75v2-media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 04:51:47,242 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/khc0dobf-media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 04:51:47,276 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2o9zlke7-media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 04:51:47,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:51,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:51:52,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:51:53,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:51:56,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:51:58,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:00,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:00,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:01,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:02,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:02,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:02,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:02,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:04,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:06,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:06,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:09,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:52:09,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:11,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:11,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:11,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:11,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:11,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:11,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:14,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:15,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:15,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:16,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:20,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:21,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:22,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:22,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:22,516 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:22,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:26,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:26,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:26,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:30,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:30,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:31,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:31,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:33,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:33,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:33,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:33,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:36,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:36,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:37,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:39,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:52:41,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:42,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:43,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:43,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:43,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:43,291 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:45,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:52:45,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:52:47,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:47,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:47,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:52,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:53,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:52:53,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:52:53,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:52:53,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:52:53,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:52:57,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:52:57,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:52:58,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:00,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:00,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:02,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:03,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:53:03,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:03,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:53:03,551 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:53:03,553 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:53:07,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:53:07,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:09,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:53:09,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:12,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:13,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:53:13,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:53:13,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:53:13,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:53:14,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:15,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:15,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:17,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:53:17,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:18,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:53:18,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:53:18,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:53:18,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:53:19,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:22,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:53:22,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:53:24,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:29,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:30,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:30,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:35,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:39,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:53:41,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:45,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:53:45,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:53:46,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:51,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:53:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:00,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:00,759 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:02,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:07,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:09,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:54:12,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_a90fd9580e1c1e24f448.png with policy now +2023-07-12 04:54:12,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_6b84704eb15e35e5f9a9.png with policy now +2023-07-12 04:54:12,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_6e5ebba16091d67e0c49.png with policy now +2023-07-12 04:54:12,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:54:12,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1160_9a7e5eb802701258688c.png with policy now +2023-07-12 04:54:12,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:12,849 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:12,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:12,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:12,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:13,149 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dbdnof94-media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 04:54:13,161 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3dkexbud-media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 04:54:13,168 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0u22r6yp-media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 04:54:13,191 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6q84o7pz-media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 04:54:15,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:15,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:17,389 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 04:54:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 04:54:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 04:54:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 04:54:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:54:17,838 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jajy3kz4-media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 04:54:17,840 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ybekej2n-media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 04:54:17,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:17,874 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p9o7tl56-media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 04:54:17,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e81ohyl3-media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 04:54:22,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:23,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:23,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:27,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:28,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:30,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:30,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:31,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:31,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:31,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:31,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:32,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:32,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:33,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:37,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:39,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:54:39,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:41,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:41,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:41,556 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:41,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:42,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:42,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:44,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:45,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:54:45,923 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:54:47,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:50,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:52,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:54:52,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:54:52,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:54:52,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:54:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:54:52,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:54:55,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:54:57,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:00,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:01,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:01,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:02,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:02,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:02,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:02,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:02,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:02,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:05,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:07,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:09,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:55:11,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:12,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:12,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:12,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:12,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:12,807 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:16,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:16,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:16,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:17,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:17,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:22,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:22,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:23,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:23,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:23,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:23,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:27,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:27,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:27,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:31,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:31,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:32,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:33,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:37,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:37,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:55:40,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:40,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:40,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:40,868 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:40,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:55:42,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:42,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:44,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:44,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:44,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:44,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:46,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:46,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:55:46,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:55:46,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:55:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:48,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:55:48,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:55:48,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:55:48,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:55:51,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:55:52,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:55:52,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:55:56,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:01,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:01,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:01,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:01,789 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:06,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:09,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:56:12,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:16,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:16,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:16,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:17,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:22,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:27,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:31,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:31,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:31,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:33,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:38,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:39,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:56:43,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_a451b10b1cddeff35521.png with policy now +2023-07-12 04:56:43,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_2bc25132345cfabe28d9.png with policy now +2023-07-12 04:56:43,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_f9f3ae84aa2fd03f0c14.png with policy now +2023-07-12 04:56:43,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:56:43,353 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1170_e76ace79b32cf06e3105.png with policy now +2023-07-12 04:56:43,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:56:43,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:56:43,395 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:56:43,397 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:56:43,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j4d6bmxw-media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 04:56:43,655 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pp5i3vsz-media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 04:56:43,691 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i09ncfdm-media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 04:56:43,727 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5oetk4ej-media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 04:56:44,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:46,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:56:46,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:56:46,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:56:47,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:56:47,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:56:47,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 04:56:47,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 04:56:47,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 04:56:47,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 04:56:47,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:56:48,251 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eh6pgpa7-media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 04:56:48,253 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/765hz2iu-media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 04:56:48,276 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i3xiuagw-media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 04:56:48,279 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iu8dp028-media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 04:56:49,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:52,930 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:56:53,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:56:55,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:56:57,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:00,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:01,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:01,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:01,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:02,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:03,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:03,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:03,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:03,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:05,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:07,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:07,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:09,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:57:10,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:12,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:13,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:13,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:13,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:13,111 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:16,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:17,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:17,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:17,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:17,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:17,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:22,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:22,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:22,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:22,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:22,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:22,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:22,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:27,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:28,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:32,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:32,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:32,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:32,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:32,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:32,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:32,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:32,949 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:33,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:33,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:38,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:38,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:39,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:57:43,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:43,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:43,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:43,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:43,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:43,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:47,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:57:47,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:57:47,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:57:48,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:48,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:49,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:53,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:53,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:57:53,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:57:53,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:57:53,347 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:57:54,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:57:58,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:57:58,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:57:59,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:02,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:02,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:02,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:03,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:03,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:58:03,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:58:03,851 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:58:03,853 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:58:04,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:08,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:58:08,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:09,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:58:10,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:13,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:14,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:58:14,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:58:14,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:58:14,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:58:15,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:17,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:17,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:17,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:18,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:58:18,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:19,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:58:19,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:58:19,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:58:19,184 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:58:20,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:23,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:58:23,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:58:25,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:30,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:32,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:32,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:32,761 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:35,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:39,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:58:41,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:46,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:47,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:58:47,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:58:47,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:58:52,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:58:57,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:02,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:02,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:02,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:02,927 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:07,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:09,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:59:13,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:13,404 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,404 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_d085f51cf12dae1345cd.png with policy now +2023-07-12 04:59:13,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_f8627735c220e9667e1c.png with policy now +2023-07-12 04:59:13,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_20948b93a6c00143c135.png with policy now +2023-07-12 04:59:13,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 04:59:13,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 04:59:13,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:59:13,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 04:59:13,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1180_7b2ab005b9837744864c.png with policy now +2023-07-12 04:59:13,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:13,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:13,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:13,503 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:13,742 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5vqejp2n-media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 04:59:13,744 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fkepbf8b-media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 04:59:13,745 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpm21z2y-media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 04:59:13,805 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o6acm935-media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 04:59:14,149 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gip21clf-media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 04:59:14,167 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wqop05je-media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 04:59:17,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:17,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:17,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:18,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 04:59:18,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 04:59:18,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:18,825 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83zuq9ip-media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 04:59:18,832 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/holu3ce4-media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 04:59:23,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:23,461 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:24,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:28,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:29,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:31,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:31,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:31,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:31,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:33,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:33,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:33,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:33,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:33,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:35,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:38,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:39,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 04:59:40,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:41,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:41,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:41,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:41,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:43,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:43,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:45,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:48,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:49,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 04:59:49,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 04:59:49,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 04:59:50,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:51,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 04:59:51,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 04:59:51,971 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 04:59:51,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 04:59:53,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 04:59:53,532 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 04:59:56,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 04:59:58,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:01,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:02,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:02,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:02,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:02,893 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:03,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:03,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:05,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:05,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:05,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:06,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:08,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:09,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:00:11,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:13,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:13,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:13,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:13,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:13,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:13,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:17,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:18,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:20,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:20,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:20,159 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:22,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:22,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:22,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:22,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:22,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:23,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:23,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:28,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:28,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:33,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:33,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:33,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:33,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:33,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:33,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:33,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:35,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:35,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:35,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:38,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:38,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:39,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:00:43,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:44,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:44,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:44,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:44,645 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:44,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:48,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:48,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:49,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:00:49,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:49,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:00:49,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:00:49,423 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:00:50,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:00:50,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:00:50,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:00:53,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:00:53,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:00:54,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:00:59,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:04,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:05,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:05,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:05,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:09,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:01:10,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:15,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:20,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:20,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:20,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:20,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:25,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:30,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:35,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:35,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:35,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:35,797 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:39,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:01:41,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:43,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_f7abacf0c65108ebd291.png with policy now +2023-07-12 05:01:43,451 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_c631ebe8975b90c43163.png with policy now +2023-07-12 05:01:43,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_138faee5a37f54662918.png with policy now +2023-07-12 05:01:43,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:01:43,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1190_597e7e8766e45fd365d0.png with policy now +2023-07-12 05:01:43,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:01:43,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:01:43,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:01:43,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:01:43,803 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpsn8kt2-media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 05:01:43,839 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/38glgnu7-media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 05:01:43,852 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grojtuti-media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 05:01:43,856 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nv49fg2e-media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 05:01:43,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:01:43,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 05:01:43,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 05:01:43,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 05:01:43,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 05:01:43,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:01:44,346 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eytsjzyp-media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 05:01:44,361 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pzxec5l9-media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 05:01:44,380 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l52pfi4j-media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 05:01:44,385 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qbdjm7bw-media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 05:01:46,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:48,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:01:50,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:01:50,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:01:50,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:01:52,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:53,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:01:54,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:01:57,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:01:59,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:02,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:02,220 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:02,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:02,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:03,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:04,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:04,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:06,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:06,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:06,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:08,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:09,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:09,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:02:12,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:12,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:12,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:12,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:14,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:14,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:14,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:19,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:19,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:21,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:21,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:21,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:22,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:22,263 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:22,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:22,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:24,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:24,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:24,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:29,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:30,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:32,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:32,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:32,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:32,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:34,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:34,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:35,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:36,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:36,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:36,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:39,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:39,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:02:40,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:42,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:42,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:42,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:42,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:44,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:44,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:46,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:49,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:51,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:52,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:02:52,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:02:52,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:02:52,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:02:52,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:02:52,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:02:52,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:02:54,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:02:54,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:02:56,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:02:59,246 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:01,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:03,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:03:03,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:03:03,455 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:03:03,457 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:03:04,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:03:04,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:07,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:07,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:07,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:07,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:09,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:10,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:03:13,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:13,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:03:13,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:03:13,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:03:13,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:03:14,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:03:14,279 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:18,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:18,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:03:18,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:03:18,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:03:18,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:03:19,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:03:19,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:22,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:22,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:23,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:24,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:03:28,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:33,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:37,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:37,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:37,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:39,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:03:39,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:44,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:49,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:03:52,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:03:52,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:03:52,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:03:55,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:00,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:05,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:07,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:07,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:07,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:09,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:04:10,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:12,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_707f52b23811c66eb81f.png with policy now +2023-07-12 05:04:12,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,820 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_095c7719d5d88d3a784e.png with policy now +2023-07-12 05:04:12,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_509f77863e0d740b58c5.png with policy now +2023-07-12 05:04:12,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:04:12,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1200_c158e5c7fd160b30b673.png with policy now +2023-07-12 05:04:12,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:12,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:12,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:12,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:13,136 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aw2kp4l1-media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 05:04:13,138 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1jtazdkv-media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 05:04:13,146 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4ggaxzog-media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 05:04:13,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mj3r3iiv-media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 05:04:14,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:14,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 05:04:14,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 05:04:14,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 05:04:14,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 05:04:14,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:04:14,861 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q33bgrej-media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 05:04:14,862 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rv4oeqtr-media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 05:04:14,931 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m5bchzqz-media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 05:04:14,942 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y52vfqh1-media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 05:04:15,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:19,531 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:21,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:22,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:22,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:22,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:22,961 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:24,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:26,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:29,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:31,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:31,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:31,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:31,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:31,920 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:34,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:34,566 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:37,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:38,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:38,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:38,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:39,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:04:39,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:41,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:41,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:41,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:41,916 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:42,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:44,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:44,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:48,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:49,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:52,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:04:52,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:04:52,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:04:52,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:04:53,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:54,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:04:54,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:04:54,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:04:54,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:04:54,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:04:58,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:04:59,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:02,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:02,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:02,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:02,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:04,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:04,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:04,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:09,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:05:09,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:10,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:10,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:10,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:10,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:05:13,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:13,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:13,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:13,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:14,754 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:14,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:15,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:19,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:21,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:23,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:23,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:23,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:23,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:24,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:24,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:25,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:25,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:25,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:05:26,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:29,784 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:32,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:33,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:33,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:33,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:33,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:34,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:34,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:37,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:39,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:05:39,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:41,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:41,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:41,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:05:42,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:44,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:44,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:44,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:44,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:44,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:44,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:48,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:48,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:05:48,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:05:48,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:05:48,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:05:49,831 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:05:49,831 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:54,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:05:54,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:05:56,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:05:56,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:05:56,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:00,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:05,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:09,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:06:10,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:11,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:11,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:11,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:16,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:21,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:26,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:26,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:26,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:27,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:32,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:37,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:39,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:06:41,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:41,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:41,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:42,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:43,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,316 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_1ba924a90889d370d938.png with policy now +2023-07-12 05:06:43,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,330 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_25ef2dfcdcaf9ff09dea.png with policy now +2023-07-12 05:06:43,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,350 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_a311e64f75624628145d.png with policy now +2023-07-12 05:06:43,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:06:43,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1210_75f318a5faeb28d48e50.png with policy now +2023-07-12 05:06:43,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:06:43,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:06:43,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:06:43,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:06:43,652 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1vgjckud-media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 05:06:43,670 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e0ofbfor-media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 05:06:43,750 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yvmyw2tk-media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 05:06:43,806 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hwp4syfg-media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 05:06:45,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:06:45,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 05:06:45,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 05:06:45,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 05:06:45,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 05:06:45,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:06:45,426 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nqxa75ww-media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 05:06:45,445 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/plwgdw74-media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 05:06:45,462 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u5504xaj-media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 05:06:45,474 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xqvv7wef-media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 05:06:48,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:50,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:06:53,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:06:53,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:06:55,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:06:56,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:06:56,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:06:56,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:06:58,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:00,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:02,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:02,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:02,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:02,352 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:04,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:05,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:05,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:09,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:07:10,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:10,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:11,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:11,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:11,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:12,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:12,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:12,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:12,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:15,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:15,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:15,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:20,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:21,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:22,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:22,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:22,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:22,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:25,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:25,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:26,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:27,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:27,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:27,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:30,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:31,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:33,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:33,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:33,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:33,694 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:35,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:35,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:37,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:39,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:07:40,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:42,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:42,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:42,477 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:42,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:44,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:44,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:44,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:44,153 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:45,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:45,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:47,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:50,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:53,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:07:53,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:07:53,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:07:53,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:07:53,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:07:55,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:07:55,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:07:58,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:07:58,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:07:58,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:07:58,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:00,327 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:03,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:08:03,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:08:03,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:08:03,619 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:08:04,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:05,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:08:05,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:09,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:08:10,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:13,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:13,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:13,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:08:13,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:08:13,942 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:08:13,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:08:13,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:08:15,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:08:15,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:15,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:18,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:08:18,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:08:18,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:08:18,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:08:20,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:08:20,375 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:08:21,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:26,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:28,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:28,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:28,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:08:31,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:36,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:39,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:08:42,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:43,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:43,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:43,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:08:47,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:52,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:57,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:08:58,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:08:58,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:08:58,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:02,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:07,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:09,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:09:13,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:13,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_1015b192327aea438f8a.png with policy now +2023-07-12 05:09:13,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_b37517440f90e43b7657.png with policy now +2023-07-12 05:09:13,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_65b3c4bdf334afcfec36.png with policy now +2023-07-12 05:09:13,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:09:13,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1220_624173894a8202fd049b.png with policy now +2023-07-12 05:09:13,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:13,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:13,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:13,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:13,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:09:13,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:09:13,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:13,685 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/98ze89pp-media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 05:09:13,696 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mtgomn5z-media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 05:09:13,758 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gckxd9fh-media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 05:09:13,760 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t0wwx2ap-media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 05:09:15,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:15,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 05:09:15,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 05:09:15,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 05:09:15,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 05:09:15,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:09:15,962 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pycqbyel-media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 05:09:15,967 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5pyta22m-media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 05:09:15,977 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vtzjj1go-media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 05:09:16,046 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vpko4ipa-media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 05:09:18,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:20,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:23,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:24,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:25,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:28,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:09:28,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:09:28,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:29,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:30,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:32,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:32,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:32,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:32,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:34,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:35,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:35,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:39,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:09:40,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:40,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:42,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:42,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:42,746 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:42,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:44,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:09:44,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:09:44,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:09:45,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:45,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:45,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:50,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:50,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:09:53,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:09:53,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:09:53,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:09:53,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:09:55,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:09:55,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:09:55,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:00,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:00,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:00,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:00,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:00,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:03,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:03,313 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:03,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:03,315 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:05,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:05,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:05,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:09,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:10:10,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:11,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:13,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:13,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:13,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:13,344 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:15,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:15,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:15,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:15,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:15,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:16,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:20,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:22,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:24,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:24,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:24,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:24,368 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:28,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:30,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:30,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:30,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:30,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:33,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:35,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:35,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:35,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:35,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:35,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:35,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:38,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:39,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:10:40,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:44,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:45,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:10:45,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:10:45,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:10:45,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:45,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:45,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:45,594 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:45,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:45,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:50,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:10:50,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:10:50,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:10:50,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:10:50,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:10:50,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:10:50,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:55,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:10:56,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:00,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:00,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:00,444 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:01,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:06,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:09,187 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:11:12,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:15,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:15,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:15,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:17,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:22,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:27,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:30,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:30,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:30,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:36,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:41,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:11:44,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:11:44,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:45,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,074 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_bed029a53fc783aa1494.png with policy now +2023-07-12 05:11:45,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,084 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_82f6db57a6a62b37c416.png with policy now +2023-07-12 05:11:45,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,094 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_0a392241e2b0e5c56de0.png with policy now +2023-07-12 05:11:45,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:11:45,104 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1230_fe7e092e619d5a60940f.png with policy now +2023-07-12 05:11:45,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:11:45,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:11:45,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:11:45,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:11:45,637 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kjyjvhd5-media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 05:11:45,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:11:45,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:11:45,796 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8rtq4xck-media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 05:11:45,848 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bcv730lg-media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 05:11:46,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:11:46,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 05:11:46,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 05:11:46,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 05:11:46,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 05:11:46,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:11:46,136 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/epdbk4qe-media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 05:11:46,669 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3ezqrcc-media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 05:11:46,683 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4k0digv4-media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 05:11:47,049 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eef0wf0q-media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 05:11:47,107 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2zwnvswl-media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 05:11:49,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:51,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:11:54,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:11:55,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:11:56,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:01,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:01,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:01,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:01,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:01,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:04,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:04,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:04,248 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:04,250 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:06,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:06,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:07,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:09,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:12:11,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:13,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:13,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:13,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:13,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:13,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:16,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:16,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:16,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:16,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:16,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:18,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:21,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:23,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:24,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:24,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:24,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:24,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:26,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:26,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:29,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:31,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:31,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:31,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:31,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:34,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:34,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:34,640 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:34,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:34,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:36,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:36,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:39,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:12:40,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:41,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:44,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:44,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:44,449 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:44,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:45,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:46,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:46,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:47,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:12:47,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:12:47,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:12:50,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:51,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:12:55,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:12:55,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:12:55,249 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:12:55,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:12:56,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:12:56,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:12:56,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:01,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:02,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:02,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:02,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:02,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:05,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:13:05,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:13:05,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:13:05,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:13:06,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:13:06,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:07,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:09,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:13:11,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:13,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:16,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:16,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:13:16,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:13:16,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:13:16,465 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:13:17,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:17,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:17,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:18,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:21,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:13:21,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:13:21,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:13:21,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:13:21,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:13:21,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:23,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:26,446 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:13:28,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:32,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:32,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:32,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:34,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:39,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:13:40,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:45,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:47,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:13:47,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:13:47,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:13:50,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:13:55,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:00,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:02,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:14:02,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:02,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:05,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:09,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:14:11,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:15,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_9038ef38623aaf3b1297.png with policy now +2023-07-12 05:14:15,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_4fdefa2382f1383aa03b.png with policy now +2023-07-12 05:14:15,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_9b87b6c7ef2509f2091e.png with policy now +2023-07-12 05:14:15,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:14:15,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1240_563721340a1b7779ebb1.png with policy now +2023-07-12 05:14:15,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:15,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:15,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:15,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:15,726 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k3rm3soh-media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 05:14:15,773 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hy24zmfj-media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 05:14:15,778 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6hysak8e-media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 05:14:15,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/inyzzxfo-media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 05:14:16,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:16,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:16,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 05:14:16,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 05:14:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 05:14:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 05:14:16,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:14:16,990 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/irks8uit-media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 05:14:17,014 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0f0rk2j4-media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 05:14:17,026 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t02k5qcj-media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 05:14:17,030 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u1urkopz-media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 05:14:17,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:14:17,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:17,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:21,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:22,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:25,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:27,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:27,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:32,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:33,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:14:33,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:33,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:33,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:35,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:35,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:35,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:35,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:38,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:39,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:14:39,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:39,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:43,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:45,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:45,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:45,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:45,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:45,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:45,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:48,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:14:48,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:14:49,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:51,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:14:54,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:14:56,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:14:56,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:14:56,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:14:56,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:14:57,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:14:57,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:00,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:03,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:03,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:03,154 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:03,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:05,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:06,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:06,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:06,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:06,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:09,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:15:09,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:09,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:11,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:15,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:16,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:16,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:16,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:16,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:16,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:18,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:18,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:18,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:22,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:22,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:22,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:26,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:26,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:26,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:26,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:27,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:28,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:28,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:33,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:33,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:33,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:33,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:34,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:36,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:36,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:36,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:36,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:38,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:39,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:15:40,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:40,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:44,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:46,202 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:47,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:47,667 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:47,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:47,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:48,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:15:48,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:15:48,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:15:49,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:52,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:52,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:52,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:15:52,219 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:15:52,219 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:15:52,221 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:15:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:15:58,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:15:58,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:15:59,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:03,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:03,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:03,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:05,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:09,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:16:11,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:16,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:18,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:18,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:18,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:21,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:26,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:31,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:33,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:33,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:36,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:39,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:16:42,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:46,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_a92796d341fe8362362e.png with policy now +2023-07-12 05:16:46,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_7e96d8ec566f76718aea.png with policy now +2023-07-12 05:16:46,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_0ffe8cb158ff46492db0.png with policy now +2023-07-12 05:16:46,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:16:46,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1250_4abee66cab5fc4486094.png with policy now +2023-07-12 05:16:46,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:16:46,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:16:46,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:16:46,861 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:16:47,087 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tjqy2dxd-media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 05:16:47,117 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0dayqkx1-media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 05:16:47,117 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9zf987a3-media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 05:16:47,119 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/83mc26vf-media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 05:16:47,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:16:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:16:48,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:16:52,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:16:52,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:16:52,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 05:16:52,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 05:16:52,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 05:16:52,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 05:16:52,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:16:52,760 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mju1skgl-media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 05:16:52,765 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uec25tgp-media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 05:16:52,777 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q3ub8ws7-media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 05:16:52,806 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6pa0jpxe-media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 05:16:53,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:16:56,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:16:58,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:16:59,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:03,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:03,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:03,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:04,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:04,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:05,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:05,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:05,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:05,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:09,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:17:10,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:10,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:10,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:15,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:16,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:16,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:16,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:16,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:16,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:16,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:18,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:18,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:18,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:21,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:22,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:26,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:26,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:26,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:26,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:26,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:28,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:28,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:31,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:33,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:33,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:33,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:34,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:37,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:37,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:37,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:37,065 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:37,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:39,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:17:40,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:40,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:42,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:46,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:47,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:47,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:47,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:47,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:47,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:48,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:17:48,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:17:48,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:17:52,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:52,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:17:52,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:57,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:17:57,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:17:57,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:17:57,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:17:58,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:17:58,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:17:58,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:03,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:03,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:03,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:03,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:04,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:07,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:18:07,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:18:07,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:18:07,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:18:08,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:09,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:18:10,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:18:10,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:13,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:16,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:17,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:18:17,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:18:17,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:18:17,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:18:18,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:18,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:18,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:19,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:22,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:18:22,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:18:22,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:18:22,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:18:22,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:18:22,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:25,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:28,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:18:30,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:33,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:33,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:33,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:35,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:39,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:18:41,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:46,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:48,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:18:48,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:18:48,629 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:18:51,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:18:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:01,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:03,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:03,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:03,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:07,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:09,204 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:19:13,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:16,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_e893310e8c740aedacd9.png with policy now +2023-07-12 05:19:16,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,473 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_3930a2a71b1d2e79ebf6.png with policy now +2023-07-12 05:19:16,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_709f8444e3a90e20584a.png with policy now +2023-07-12 05:19:16,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:19:16,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1260_52dd5c4bac63f8392a24.png with policy now +2023-07-12 05:19:16,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:16,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:16,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:16,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:16,737 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/guhsmjtl-media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 05:19:16,778 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eo1jbvxp-media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 05:19:16,789 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bcsd5g3b-media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 05:19:16,811 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fs1amvrv-media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 05:19:16,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:16,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 05:19:16,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 05:19:16,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 05:19:16,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 05:19:16,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:19:17,247 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yjmo5jrs-media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 05:19:17,255 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9iszdyi-media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 05:19:17,275 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/svetvfjb-media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 05:19:17,306 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3s5r03yp-media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 05:19:18,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:19,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:19,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:19,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:22,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:24,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:26,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:28,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:29,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:34,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:34,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:34,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:34,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:34,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:36,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:36,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:36,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:36,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:39,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:19:39,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:40,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:40,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:44,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:46,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:46,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:46,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:46,393 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:46,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:46,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:49,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:19:49,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:19:49,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:19:50,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:52,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:19:55,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:19:56,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:19:56,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:19:56,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:19:56,191 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:19:58,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:19:58,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:00,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:04,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:04,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:04,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:04,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:06,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:06,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:06,335 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:06,335 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:06,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:09,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:20:10,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:10,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:11,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:16,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:16,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:16,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:16,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:16,880 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:17,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:17,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:19,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:19,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:19,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:21,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:23,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:26,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:27,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:27,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:27,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:27,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:29,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:29,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:32,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:34,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:34,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:34,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:35,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:37,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:37,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:37,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:37,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:37,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:39,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:20:41,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:41,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:42,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:47,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:47,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:47,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:47,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:47,676 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:47,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:49,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:20:49,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:20:49,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:20:52,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:20:52,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:20:52,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:20:52,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:20:53,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:53,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:20:53,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:20:58,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:20:59,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:21:03,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:04,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:04,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:04,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:09,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:09,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:21:14,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:19,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:19,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:19,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:19,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:24,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:29,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:34,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:35,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:35,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:35,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:39,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:21:40,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:45,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:47,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_39451786e3d6986a15e5.png with policy now +2023-07-12 05:21:47,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_7507600afa47a7721bc0.png with policy now +2023-07-12 05:21:47,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_239d53a80d648e3b5d74.png with policy now +2023-07-12 05:21:47,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:21:47,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1270_007f79d4a566d7c33ea0.png with policy now +2023-07-12 05:21:47,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:21:47,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:21:47,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:21:47,212 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:21:47,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:21:47,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 05:21:47,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 05:21:47,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 05:21:47,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 05:21:47,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:21:47,449 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/73xoa915-media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 05:21:47,458 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/185yktdd-media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 05:21:47,463 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g7syc8mb-media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 05:21:47,526 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p9pw1med-media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 05:21:47,870 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zw021il2-media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 05:21:47,894 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vybt3p8r-media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 05:21:48,206 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4vvobbx8-media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 05:21:48,271 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/us08xq4x-media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 05:21:50,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:21:50,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:21:50,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:21:50,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:53,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:21:55,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:21:57,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:21:59,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:01,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:05,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:05,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:05,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:05,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:06,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:06,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:06,402 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:06,404 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:07,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:09,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:22:11,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:11,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:12,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:15,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:15,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:15,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:15,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:17,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:17,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:18,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:20,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:20,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:20,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:23,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:24,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:25,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:25,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:25,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:25,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:29,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:29,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:29,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:35,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:35,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:35,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:35,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:35,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:35,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:35,525 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:35,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:39,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:22:40,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:41,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:41,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:45,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:45,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:45,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:45,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:45,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:47,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:47,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:50,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:22:50,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:22:50,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:22:50,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:53,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:22:55,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:22:55,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:22:55,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:22:55,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:22:56,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:22:59,639 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:22:59,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:02,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:05,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:23:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:05,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:06,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:23:06,426 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:23:06,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:23:06,428 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:23:07,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:09,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:23:11,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:23:11,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:12,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:16,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:23:16,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:23:16,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:23:16,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:23:17,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:23:17,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:17,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:20,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:20,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:21,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:23:21,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:23:21,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:23:21,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:23:23,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:23,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:23:23,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:28,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:29,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:23:33,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:35,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:23:35,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:35,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:38,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:39,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:23:44,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:49,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:50,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:23:50,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:23:50,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:23:54,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:23:59,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:04,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:05,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:24:05,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:09,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:24:10,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:15,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:16,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_91a32c15ebfc912c2201.png with policy now +2023-07-12 05:24:16,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_76c909633b7c82a1c57d.png with policy now +2023-07-12 05:24:16,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_d09f421be121a991d346.png with policy now +2023-07-12 05:24:16,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:24:16,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1280_f520f48ddc3348a6c6bd.png with policy now +2023-07-12 05:24:16,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:16,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:16,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:16,206 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:16,434 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fqhebolb-media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 05:24:16,467 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uucsgcb3-media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 05:24:16,469 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/utxhs1iq-media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 05:24:16,480 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1mdk3wfy-media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 05:24:17,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:17,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 05:24:17,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 05:24:17,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 05:24:17,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 05:24:17,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:24:18,247 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gc8za9ny-media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 05:24:18,257 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rlfhdqh6-media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 05:24:18,266 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7fpujty9-media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 05:24:18,274 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x7fg14xo-media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 05:24:20,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:20,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:20,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:23,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:25,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:26,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:29,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:31,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:35,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:35,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:35,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:35,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:35,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:35,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:35,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:35,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:36,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:39,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:24:41,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:42,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:44,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:44,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:44,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:44,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:47,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:47,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:47,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:51,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:24:51,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:24:51,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:24:53,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:53,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:24:54,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:24:54,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:24:54,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:24:54,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:24:58,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:24:59,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:24:59,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:03,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:04,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:04,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:04,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:04,340 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:05,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:05,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:06,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:06,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:06,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:09,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:09,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:25:11,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:14,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:14,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:14,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:14,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:14,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:18,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:18,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:19,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:21,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:21,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:21,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:24,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:25,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:25,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:25,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:25,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:25,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:30,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:30,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:30,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:36,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:36,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:36,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:36,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:36,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:36,403 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:36,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:36,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:36,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:39,219 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:25:42,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:42,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:42,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:46,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:46,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:46,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:46,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:47,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:48,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:48,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:51,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:25:51,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:25:51,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:25:51,083 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:25:51,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:25:51,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:25:51,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:25:53,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:25:54,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:25:54,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:25:58,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:03,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:07,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:26:07,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:07,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:08,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:09,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:26:13,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:18,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:22,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:26:22,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:22,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:24,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:29,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:34,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:37,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:37,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:26:37,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:39,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:26:39,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:44,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:45,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_c67265e3a1dfdfb99387.png with policy now +2023-07-12 05:26:45,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_e8b496a8b73ca169dce8.png with policy now +2023-07-12 05:26:45,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_ad7f6abdff5cb1890622.png with policy now +2023-07-12 05:26:45,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:26:45,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1290_499d8794ae851f2ca7c3.png with policy now +2023-07-12 05:26:45,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:26:45,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:26:45,732 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:26:45,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:26:46,036 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xyiubq3i-media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 05:26:46,047 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pojd9kfo-media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 05:26:46,054 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dt801efh-media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 05:26:46,059 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kur41fri-media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 05:26:48,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:26:48,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 05:26:48,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 05:26:48,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 05:26:48,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 05:26:48,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:26:48,729 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i61xmc7f-media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 05:26:48,734 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sq4bx3wb-media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 05:26:48,767 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/updq52uf-media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 05:26:48,777 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mtr3i4sb-media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 05:26:49,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:52,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:26:52,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:26:54,408 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:26:54,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:26:54,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:00,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:00,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:03,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:03,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:03,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:03,950 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:05,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:06,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:06,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:07,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:07,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:07,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:09,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:27:10,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:12,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:13,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:13,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:13,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:13,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:16,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:18,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:18,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:22,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:23,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:23,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:23,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:23,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:23,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:23,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:24,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:24,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:27,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:30,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:32,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:34,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:34,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:34,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:34,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:36,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:36,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:38,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:38,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:38,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:38,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:39,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:27:42,503 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:43,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:45,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:45,262 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:45,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:45,264 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:27:48,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:27:48,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:49,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:53,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:27:53,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:27:53,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:27:54,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:27:55,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:27:55,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:27:55,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:27:55,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:27:55,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:00,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:00,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:00,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:06,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:28:06,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:06,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:28:06,002 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:28:06,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:06,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:06,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:08,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:08,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:08,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:09,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:28:11,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:12,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:16,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:28:16,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:28:16,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:28:16,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:17,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:18,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:18,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:20,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:28:20,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:28:20,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:28:20,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:28:22,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:23,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:23,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:24,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:28:24,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:28:28,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:33,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:38,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:38,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:38,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:38,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:39,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:28:44,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:49,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:53,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:28:53,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:28:53,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:28:54,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:28:59,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:04,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:08,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:08,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:08,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:09,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:29:10,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:15,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:15,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,381 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_6d25eb3589d810ed3440.png with policy now +2023-07-12 05:29:15,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_295181c2c8f74bdb7122.png with policy now +2023-07-12 05:29:15,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_d49ed1edb101f8e16c80.png with policy now +2023-07-12 05:29:15,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:29:15,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1300_47d0444bbe383a2f3ed2.png with policy now +2023-07-12 05:29:15,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:15,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:15,455 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:15,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:15,692 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mgp1sv3d-media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 05:29:15,697 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpwjknuj-media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 05:29:15,715 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/elbxqpu0-media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 05:29:15,727 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5svabk9h-media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 05:29:18,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:18,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 05:29:18,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 05:29:18,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 05:29:18,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 05:29:18,887 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:29:19,220 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5mjyq87i-media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 05:29:19,221 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o4y5r6fe-media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 05:29:19,224 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/es4sx1bs-media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 05:29:19,227 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ko16iiry-media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 05:29:20,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:23,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:23,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:23,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:24,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:24,895 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:26,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:30,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:31,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:34,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:34,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:34,208 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:34,210 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:36,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:36,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:36,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:38,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:38,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:38,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:39,228 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:29:41,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:42,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:44,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:44,107 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:44,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:44,109 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:47,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:48,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:48,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:52,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:29:53,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:29:53,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:29:53,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:29:54,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:29:54,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:29:54,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:29:54,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:29:54,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:29:54,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:29:57,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:00,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:03,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:04,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:04,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:04,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:04,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:06,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:06,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:08,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:08,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:30:08,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:08,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:09,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:30:12,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:14,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:14,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:14,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:14,631 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:14,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:19,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:19,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:19,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:24,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:30:24,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:24,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:24,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:24,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:24,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:24,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:24,676 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:25,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:25,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:29,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:31,114 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:35,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:35,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:35,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:35,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:35,704 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:37,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:37,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:39,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:39,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:30:39,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:39,231 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:30:41,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:43,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:46,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:46,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:46,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:46,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:47,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:49,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:49,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:50,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:30:50,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:30:50,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:30:50,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:30:52,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:30:54,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:30:54,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:30:55,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:30:55,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:30:58,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:03,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:08,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:09,232 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:31:09,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:09,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:09,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:13,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:18,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:23,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:24,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:24,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:24,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:28,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:33,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:38,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:39,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:31:39,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:39,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:39,448 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:43,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:44,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_5c747b5d08a35792803f.png with policy now +2023-07-12 05:31:44,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_8d7535a093e8d31d2071.png with policy now +2023-07-12 05:31:44,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_b02cf0aa477e22c8930d.png with policy now +2023-07-12 05:31:44,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:31:44,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1310_59c1d57a32cc82a9a3ff.png with policy now +2023-07-12 05:31:45,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:31:45,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:31:45,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:31:45,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:31:45,261 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6hzrl1jo-media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 05:31:45,268 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1eyk0t11-media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 05:31:45,273 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4g9vpns0-media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 05:31:45,276 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k89r9ofs-media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 05:31:48,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:49,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:31:49,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:31:49,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 05:31:49,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 05:31:49,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 05:31:49,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 05:31:49,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:31:49,723 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lo2yfro2-media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 05:31:49,734 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7rk6srp-media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 05:31:49,748 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0b6hbuvr-media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 05:31:49,753 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/98sbv1ki-media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 05:31:53,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:31:54,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:31:54,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:31:54,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:31:54,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:31:55,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:31:59,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:01,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:04,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:04,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:04,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:04,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:04,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:07,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:07,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:09,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:32:09,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:09,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:09,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:09,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:13,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:14,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:14,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:14,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:14,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:14,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:19,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:19,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:19,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:24,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:24,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:24,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:24,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:24,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:24,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:24,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:25,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:25,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:25,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:30,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:31,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:34,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:34,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:34,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:34,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:36,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:37,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:37,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:39,235 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:32:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:39,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:41,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:43,459 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:44,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:44,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:44,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:44,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:46,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:49,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:49,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:52,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:32:54,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:32:54,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:32:54,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:32:55,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:32:55,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:32:55,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:32:55,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:32:55,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:32:55,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:32:57,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:01,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:02,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:05,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:33:05,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:33:05,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:33:05,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:33:07,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:33:07,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:07,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:09,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:33:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:33:09,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:09,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:13,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:13,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:16,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:33:16,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:33:16,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:33:16,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:33:18,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:19,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:33:19,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:20,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:33:20,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:33:20,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:33:20,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:33:24,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:24,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:24,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:25,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:33:25,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:33:29,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:34,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:39,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:33:39,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:33:39,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:39,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:42,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:47,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:52,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:33:54,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:33:54,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:33:54,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:33:58,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:03,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:08,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:09,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:34:10,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:10,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:10,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:13,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:15,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_e5cb375d6da828cc3e3e.png with policy now +2023-07-12 05:34:15,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_064f388777b3203b13c5.png with policy now +2023-07-12 05:34:15,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_a24af0c79adf71caef03.png with policy now +2023-07-12 05:34:15,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:34:15,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1320_9bd0004be4c4a0c4f009.png with policy now +2023-07-12 05:34:15,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:15,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:15,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:15,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:15,404 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z09ayj1o-media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 05:34:15,441 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fc3yp05y-media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 05:34:15,468 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/de38dtsn-media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 05:34:15,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8dzvobj5-media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 05:34:18,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:19,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:19,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:19,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 05:34:19,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 05:34:19,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 05:34:19,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 05:34:19,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:34:20,185 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8e9il45f-media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 05:34:20,186 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ruiyhxl7-media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 05:34:20,209 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b96xpv4i-media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 05:34:20,214 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xh95u0bo-media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 05:34:24,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:24,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:25,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:25,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:25,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:25,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:29,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:31,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:33,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:33,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:33,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:33,634 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:35,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:37,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:37,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:39,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:34:40,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:40,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:40,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:40,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:43,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:43,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:43,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:43,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:43,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:43,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:46,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:49,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:52,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:34:54,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:34:54,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:34:54,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:34:54,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:34:55,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:34:55,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:34:56,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:34:56,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:34:56,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:34:57,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:01,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:03,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:04,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:04,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:04,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:04,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:07,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:07,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:09,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:35:09,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:11,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:11,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:11,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:35:13,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:14,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:14,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:14,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:14,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:15,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:20,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:20,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:20,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:25,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:25,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:25,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:25,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:26,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:26,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:26,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:27,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:27,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:27,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:35:31,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:32,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:35,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:35,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:35,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:35,520 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:36,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:38,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:38,127 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:39,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:35:41,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:43,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:43,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:43,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:35:44,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:45,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:45,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:45,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:45,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:46,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:50,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:50,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:50,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:35:50,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:35:50,720 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:35:50,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:35:52,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:56,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:35:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:35:57,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:35:58,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:35:58,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:35:58,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:02,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:07,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:09,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:36:13,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:13,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:13,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:36:13,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:18,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:23,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:28,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:28,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:29,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:34,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:39,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:36:40,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:36:43,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:36:45,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_b1042365de64e0a942b0.png with policy now +2023-07-12 05:36:45,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_dbe013ae177630d5a987.png with policy now +2023-07-12 05:36:45,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_3bca8acf12b4c46ab3ce.png with policy now +2023-07-12 05:36:45,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:36:45,103 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1330_473197dd1f22461890d6.png with policy now +2023-07-12 05:36:45,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:36:45,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:36:45,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:36:45,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:36:45,417 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/trgvsd55-media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 05:36:45,442 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mtidgjlq-media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 05:36:45,448 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n361rlu2-media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 05:36:45,470 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5mzi9b6h-media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 05:36:46,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 05:36:50,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 05:36:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 05:36:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 05:36:50,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:36:50,736 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k0i8s6c9-media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 05:36:50,764 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nate0cs8-media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 05:36:50,798 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9gmwzm79-media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 05:36:50,864 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d895tav2-media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 05:36:51,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:55,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:36:56,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:36:57,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:36:58,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:36:58,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:02,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:02,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:04,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:04,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:04,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:04,267 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:07,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:08,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:08,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:09,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:37:13,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:13,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:13,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:13,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:13,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:13,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:13,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:14,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:14,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:18,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:20,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:24,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:24,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:24,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:24,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:24,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:26,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:26,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:28,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:28,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:37:28,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:29,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:32,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:35,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:35,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:35,288 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:35,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:35,291 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:38,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:38,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:39,248 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:37:41,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:44,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:44,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:37:44,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:44,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:37:45,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:45,860 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:45,861 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:45,863 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:46,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:50,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:50,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:52,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:55,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:37:55,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:37:55,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:37:55,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:37:56,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:37:56,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:37:57,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:37:59,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:37:59,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:37:59,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:02,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:03,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:05,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:38:05,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:38:05,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:38:05,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:38:08,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:38:08,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:09,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:09,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:38:14,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:14,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:38:14,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:38:14,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:14,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:16,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:38:16,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:38:16,671 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:38:16,673 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:38:20,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:20,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:38:20,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:21,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:38:21,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:38:21,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:38:21,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:38:25,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:26,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:38:26,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:38:29,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:38:29,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:38:29,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:31,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:36,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:39,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:38:42,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:44,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:38:44,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:38:44,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:38:47,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:52,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:38:57,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:00,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:00,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:00,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:03,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:08,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:09,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:39:13,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:14,935 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_87ea3e96f26f5ebb57af.png with policy now +2023-07-12 05:39:14,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_ccdaf0fc6a17d6760008.png with policy now +2023-07-12 05:39:14,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_ff1f03141cde2a233297.png with policy now +2023-07-12 05:39:14,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:39:14,965 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1340_a1fff07a952b497ecc10.png with policy now +2023-07-12 05:39:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:15,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:15,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:15,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:15,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:15,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:15,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:15,220 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oicjjrvp-media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 05:39:15,247 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d36xcn2o-media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 05:39:15,255 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/51x666h9-media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 05:39:15,278 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/baq6vdhk-media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 05:39:18,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:20,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:20,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:20,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 05:39:20,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 05:39:20,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 05:39:20,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 05:39:20,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:39:21,218 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sj6f18di-media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 05:39:21,222 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ta3e5y9t-media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 05:39:21,223 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhbaxaw6-media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 05:39:21,229 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zvf2zm7g-media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 05:39:24,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:25,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:26,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:29,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:30,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:30,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:30,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:32,899 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:35,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:35,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:35,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:35,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:35,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:38,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:38,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:39,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:39:40,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:44,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:45,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:39:45,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:39:45,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:39:45,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:46,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:46,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:46,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:46,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:50,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:50,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:50,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:39:56,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:39:56,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:39:56,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:39:56,179 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:39:56,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:39:56,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:39:56,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:00,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:00,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:00,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:02,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:06,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:06,197 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:06,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:06,199 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:07,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:08,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:08,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:09,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:40:12,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:14,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:15,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:15,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:15,845 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:16,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:16,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:16,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:16,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:18,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:21,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:21,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:23,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:26,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:26,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:26,336 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:26,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:27,115 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:27,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:29,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:31,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:31,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:31,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:33,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:34,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:37,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:37,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:37,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:37,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:39,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:39,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:39,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:40:39,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:45,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:45,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:46,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:40:46,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:40:46,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:40:46,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:46,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:46,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:46,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:51,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:51,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:51,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:40:51,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:40:51,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:40:51,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:40:51,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:40:57,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:40:57,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:40:57,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:41:01,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:01,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:01,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:02,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:07,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:09,255 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:41:13,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:16,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:16,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:16,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:18,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:23,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:28,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:31,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:31,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:31,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:33,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:38,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:39,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:41:44,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:45,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_f98823b83c4cee5e89ac.png with policy now +2023-07-12 05:41:45,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_8b4cb7fa6ec33b1868bc.png with policy now +2023-07-12 05:41:45,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_8fdfffa2228e3683ab99.png with policy now +2023-07-12 05:41:45,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:41:45,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1350_e32477931d4fd98f823b.png with policy now +2023-07-12 05:41:45,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:41:45,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:41:45,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:41:45,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:41:46,124 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i66oqag7-media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 05:41:46,176 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1wiur99q-media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 05:41:46,177 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k0aqo5fp-media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 05:41:46,179 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dmdilxuj-media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 05:41:46,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:41:46,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:41:46,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:41:50,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:51,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:41:51,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 05:41:51,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 05:41:51,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 05:41:51,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 05:41:51,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:41:51,762 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uk7s2q91-media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 05:41:51,763 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a4q0q80z-media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 05:41:51,769 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r3n8uqve-media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 05:41:51,821 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jvx5xwz6-media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 05:41:56,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:41:56,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:41:57,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:01,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:01,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:01,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:01,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:03,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:05,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:05,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:05,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:05,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:06,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:09,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:42:09,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:09,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:12,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:15,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:15,431 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:15,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:15,433 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:15,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:15,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:17,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:17,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:17,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:17,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:21,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:22,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:25,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:25,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:25,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:25,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:27,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:27,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:27,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:32,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:32,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:32,843 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:32,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:33,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:36,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:36,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:36,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:36,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:38,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:39,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:42:39,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:39,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:43,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:45,503 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:45,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:45,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:45,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:45,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:48,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:42:48,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:42:48,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:42:48,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:51,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:51,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:53,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:42:56,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:42:56,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:42:56,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:42:56,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:42:57,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:42:57,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:42:59,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:03,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:03,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:03,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:03,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:05,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:06,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:43:06,468 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:43:06,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:43:06,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:43:09,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:43:09,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:43:09,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:10,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:15,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:16,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:17,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:43:17,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:43:17,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:43:17,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:43:18,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:18,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:18,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:21,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:43:21,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:21,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:22,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:43:22,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:43:22,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:43:22,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:43:27,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:43:27,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:43:27,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:32,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:33,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:33,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:33,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:38,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:39,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:43:44,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:48,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:43:48,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:43:48,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:43:49,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:54,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:43:59,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:03,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:03,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:03,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:04,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:09,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:44:10,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:15,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:16,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_49ed245ee23ca275689e.png with policy now +2023-07-12 05:44:16,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_7925401a6a5e9671ca9b.png with policy now +2023-07-12 05:44:16,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_e9a621d16e71d8a70ce6.png with policy now +2023-07-12 05:44:16,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:44:16,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1360_fea5b00a54b6cde2ffad.png with policy now +2023-07-12 05:44:16,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:16,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:16,739 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:16,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:16,967 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k3fc7efp-media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 05:44:16,977 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vfy9t98q-media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 05:44:16,985 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qvryl6z1-media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 05:44:17,011 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gftkzknp-media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 05:44:18,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:18,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:18,800 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:20,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:21,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:21,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:21,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 05:44:21,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 05:44:21,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 05:44:21,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 05:44:21,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:44:22,342 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/luxrzb9j-media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 05:44:22,359 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ilaz72yt-media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 05:44:22,360 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zrhn2rgh-media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 05:44:22,366 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5xbfyc3e-media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 05:44:26,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:26,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:27,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:31,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:33,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:33,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:33,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:33,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:36,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:36,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:36,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:36,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:36,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:39,261 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:44:39,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:39,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:41,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:45,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:47,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:47,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:47,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:47,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:47,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:48,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:44:48,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:44:48,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:44:51,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:51,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:53,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:44:57,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:44:57,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:44:57,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:44:57,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:44:58,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:44:58,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:44:58,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:03,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:03,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:45:03,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:03,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:04,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:08,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:08,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:08,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:08,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:09,263 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:45:09,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:10,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:10,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:14,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:16,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:18,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:18,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:18,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:18,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:18,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:18,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:45:18,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:20,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:22,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:22,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:25,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:28,173 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:28,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:28,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:28,880 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:28,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:33,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:33,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:34,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:34,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:36,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:39,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:45:39,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:39,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:39,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:39,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:40,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:40,199 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:41,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:46,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:46,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:48,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:45:48,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:45:48,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:45:49,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:49,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:49,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:49,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:51,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:52,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:52,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:45:54,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:45:54,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:45:54,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:45:54,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:45:57,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:45:58,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:45:58,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:46:02,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:03,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:03,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:03,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:08,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:09,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:46:13,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:18,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:19,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:19,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:19,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:24,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:29,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:34,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:34,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:34,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:34,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:39,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:46:39,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:44,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:48,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_482ad8df2ce841450721.png with policy now +2023-07-12 05:46:48,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_4e545b1d5aa48bae02e7.png with policy now +2023-07-12 05:46:48,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_d6a886a91a9a1c97ff8e.png with policy now +2023-07-12 05:46:48,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:46:48,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1370_e377b1a6785b8362f8b8.png with policy now +2023-07-12 05:46:48,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:46:48,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:46:48,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:46:48,617 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:46:48,840 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/98s6nbmv-media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 05:46:48,869 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zssq4f35-media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 05:46:48,883 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/foqgvxvd-media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 05:46:49,070 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rauoneb8-media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 05:46:49,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:46:49,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:46:49,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:46:50,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:52,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:46:52,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 05:46:52,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 05:46:52,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 05:46:52,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 05:46:52,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:46:52,806 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zwzkrytu-media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 05:46:52,820 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/na7ko6dv-media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 05:46:52,824 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/39810n4h-media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 05:46:52,827 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0pnkliau-media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 05:46:56,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:46:58,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:46:58,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:01,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:04,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:04,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:04,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:04,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:06,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:08,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:08,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:08,132 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:08,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:09,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:47:10,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:10,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:12,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:16,501 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:17,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:17,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:17,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:17,584 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:18,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:19,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:19,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:19,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:22,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:22,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:24,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:28,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:28,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:28,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:28,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:28,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:28,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:30,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:34,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:36,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:36,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:36,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:39,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:47:39,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:39,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:39,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:39,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:40,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:40,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:40,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:46,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:46,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:48,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:48,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:48,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:48,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:51,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:47:51,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:47:51,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:47:51,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:52,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:52,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:47:57,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:47:58,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:47:58,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:47:58,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:47:58,542 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:47:58,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:47:58,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:02,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:04,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:06,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:06,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:06,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:07,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:08,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:48:08,644 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:48:08,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:48:08,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:48:09,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:48:10,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:48:10,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:12,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:16,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:18,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:19,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:48:19,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:48:19,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:48:19,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:48:21,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:21,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:21,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:22,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:48:22,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:24,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:24,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:48:24,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:48:24,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:48:24,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:48:28,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:48:28,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:48:29,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:34,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:36,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:36,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:36,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:39,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:48:40,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:45,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:50,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:48:51,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:48:51,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:48:51,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:48:55,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:00,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:05,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:06,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:06,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:06,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:09,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:49:11,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:16,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:18,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_ba0b7fdc45db85580b1b.png with policy now +2023-07-12 05:49:18,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_d46d1a613eb099818463.png with policy now +2023-07-12 05:49:18,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_18b8edd8e82489c2e700.png with policy now +2023-07-12 05:49:18,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:49:18,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1380_b7ab393ffe13b943540d.png with policy now +2023-07-12 05:49:18,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:18,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:18,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:18,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:18,483 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/unyb8m4b-media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 05:49:18,529 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d9ykmmtt-media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 05:49:18,561 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j5dvpws8-media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 05:49:18,604 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/roa0epyo-media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 05:49:21,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:21,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:21,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:21,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:22,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:22,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:22,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 05:49:22,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 05:49:22,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 05:49:22,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 05:49:22,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:49:23,334 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pw01x64b-media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 05:49:23,341 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xj3u4pzm-media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 05:49:23,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27rl1os2-media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 05:49:23,359 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/im67pzt9-media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 05:49:27,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:28,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:29,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:33,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:35,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:36,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:36,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:36,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:38,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:38,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:38,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:38,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:39,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:39,273 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:49:41,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:41,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:44,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:47,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:47,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:47,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:47,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:47,746 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:49,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:52,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:49:52,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:49:52,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:49:53,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:53,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:49:55,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:49:58,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:49:58,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:49:58,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:49:58,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:49:59,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:49:59,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:00,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:05,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:06,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:08,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:08,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:08,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:09,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:50:09,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:09,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:09,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:09,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:11,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:11,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:11,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:16,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:17,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:19,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:19,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:19,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:19,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:21,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:23,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:23,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:23,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:23,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:23,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:26,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:29,219 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:29,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:29,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:29,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:29,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:32,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:35,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:35,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:38,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:38,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:38,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:38,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:39,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:39,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:39,155 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:39,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:39,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:50:41,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:41,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:44,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:47,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:48,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:48,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:48,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:48,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:49,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:53,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:53,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:53,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:50:53,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:50:53,308 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:50:53,310 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:50:53,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:50:53,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:50:53,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:50:54,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:50:59,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:50:59,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:50:59,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:04,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:08,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:08,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:08,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:09,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:51:10,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:15,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:20,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:23,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:23,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:23,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:25,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:30,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:35,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:39,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:39,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:39,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:39,981 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:51:40,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:45,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:47,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_2048ef7db009bfd7c05d.png with policy now +2023-07-12 05:51:47,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,750 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_02fa9f2ed8ff12c99aa7.png with policy now +2023-07-12 05:51:47,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_fb6772a80e5d23777bc1.png with policy now +2023-07-12 05:51:47,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:51:47,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1390_506d872797968a090e00.png with policy now +2023-07-12 05:51:47,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:51:47,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:51:47,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:51:47,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:51:48,043 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bp7aptio-media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 05:51:48,045 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r7xipqxk-media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 05:51:48,058 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7z3iszbl-media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 05:51:48,098 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5p3dn42s-media/images/validation_1390_506d872797968a090e00.png +2023-07-12 05:51:51,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:53,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:51:53,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:51:53,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 05:51:53,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_506d872797968a090e00.png +2023-07-12 05:51:53,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 05:51:53,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 05:51:53,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:51:53,836 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/54bnzhan-media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 05:51:53,842 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2en5mv0o-media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 05:51:53,843 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nu9csk0j-media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 05:51:53,845 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/70rcr4ch-media/images/validation_1390_506d872797968a090e00.png +2023-07-12 05:51:54,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:51:54,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:51:54,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:51:56,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:51:58,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:51:59,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:01,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:05,503 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:07,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:07,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:07,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:07,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:07,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:09,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:52:09,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:09,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:09,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:11,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:11,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:12,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:17,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:17,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:17,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:17,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:17,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:17,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:17,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:22,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:23,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:24,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:24,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:24,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:26,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:26,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:26,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:26,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:28,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:29,550 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:29,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:33,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:35,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:36,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:36,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:36,828 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:36,830 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:38,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:39,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:52:39,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:39,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:39,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:41,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:41,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:44,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:47,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:47,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:47,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:47,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:47,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:47,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:49,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:53,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:52:54,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:52:54,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:52:54,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:52:54,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:52:57,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:52:57,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:52:57,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:52:57,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:52:59,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:52:59,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:00,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:05,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:06,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:07,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:53:07,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:53:07,444 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:53:07,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:53:09,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:53:09,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:09,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:09,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:53:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:12,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:17,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:17,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:53:17,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:53:17,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:53:17,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:53:17,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:53:17,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:22,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:53:22,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:22,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:53:22,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:53:22,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:53:23,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:53:23,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:24,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:24,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:24,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:27,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:29,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:53:32,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:37,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:39,283 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:53:39,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:39,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:39,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:42,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:47,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:52,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:53:54,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:53:54,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:53:54,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:53:58,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:03,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:08,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:09,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:54:09,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:09,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:09,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:13,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:16,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,769 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_66aa1d0173b44509cd1d.png with policy now +2023-07-12 05:54:16,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_e84af5fa7376533300be.png with policy now +2023-07-12 05:54:16,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,810 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_f29eae3956b8b680487b.png with policy now +2023-07-12 05:54:16,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:54:16,811 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1400_d020640095c66cf43293.png with policy now +2023-07-12 05:54:16,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:16,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:16,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:16,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:54:17,080 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cvyixbfn-media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 05:54:17,101 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqffnhpz-media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 05:54:17,139 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t40ljbf4-media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 05:54:17,149 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r2n3m1j1-media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 05:54:18,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:54:18,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 05:54:18,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 05:54:18,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 05:54:18,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 05:54:18,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:54:18,374 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mr2fk0wv-media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 05:54:18,393 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/so5kups0-media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 05:54:18,394 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gf9js64d-media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 05:54:18,400 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/694at0by-media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 05:54:18,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:23,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:24,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:24,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:24,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:24,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:26,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:29,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:30,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:34,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:35,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:35,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:35,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:35,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:54:36,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:54:36,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:39,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:54:40,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:40,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:40,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:40,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:42,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:45,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:46,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:46,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:46,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:46,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:54:48,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:54:48,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:51,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:54,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:54:56,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:54:56,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:54:56,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:54:56,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:54:56,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:54:56,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:54:56,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:54:56,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:00,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:00,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:02,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:06,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:07,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:07,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:07,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:07,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:07,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:09,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:55:12,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:12,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:12,079 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:12,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:12,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:12,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:17,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:17,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:17,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:17,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:17,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:18,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:18,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:23,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:24,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:27,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:27,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:27,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:27,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:27,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:27,778 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:27,780 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:29,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:30,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:30,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:34,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:36,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:37,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:37,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:37,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:37,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:39,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:55:39,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:42,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:42,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:42,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:42,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:42,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:45,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:48,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:48,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:48,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:48,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:48,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:48,581 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:51,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:55:52,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:55:52,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:55:52,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:55:52,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:55:54,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:55:54,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:55:57,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:55:57,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:55:57,525 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:55:57,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:00,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:56:02,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:07,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:09,288 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:56:12,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:12,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:12,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:12,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:17,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:22,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:27,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:27,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:27,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:27,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:32,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:37,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:39,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:56:42,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:42,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:42,961 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:42,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:47,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,126 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_89c2655d6ea12bc678a5.png with policy now +2023-07-12 05:56:47,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_754b2c1e14f253b4c743.png with policy now +2023-07-12 05:56:47,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_0de4ce377ce0650003a1.png with policy now +2023-07-12 05:56:47,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:56:47,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1410_57eaa364b09be7b1c708.png with policy now +2023-07-12 05:56:47,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:56:47,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:56:47,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:56:47,208 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:56:47,443 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/si3snp5r-media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 05:56:47,474 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3z1auwza-media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 05:56:47,499 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5x8c3jho-media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 05:56:47,504 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3j49o5ps-media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 05:56:48,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:48,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:56:48,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 05:56:48,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 05:56:48,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 05:56:48,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 05:56:48,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:56:49,306 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5zk2sxzl-media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 05:56:49,350 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gth5ewve-media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 05:56:49,354 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ywuw4uk-media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 05:56:49,462 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3zjuqz6b-media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 05:56:53,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:56:54,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:56:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:56:58,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:56:58,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:56:58,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:56:58,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:00,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:03,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:06,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:06,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:06,427 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:06,429 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:06,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:06,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:09,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:09,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:57:12,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:13,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:57:13,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:13,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:57:14,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:17,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:17,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:17,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:17,026 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:18,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:18,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:19,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:24,897 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:25,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:27,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:27,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:27,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:27,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:28,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:28,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:57:30,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:30,910 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:30,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:35,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:36,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:38,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:38,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:38,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:38,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:39,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:57:41,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:42,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:42,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:43,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:43,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:57:43,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:57:46,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:48,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:48,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:48,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:48,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:49,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:57:49,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:52,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:55,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:57:57,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:57:58,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:57:58,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:57:58,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:57:58,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:57:58,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:57:58,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:01,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:58:01,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:03,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:07,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:08,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:58:08,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:08,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:58:08,613 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:58:08,615 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:58:09,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:58:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:58:13,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:13,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:13,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:13,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:19,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:19,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:58:19,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:58:19,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:19,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:58:19,374 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:58:24,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:58:24,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:58:24,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:58:24,018 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:58:24,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:25,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:58:25,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:58:28,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:28,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:29,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:34,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:39,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:58:40,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:43,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:58:43,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:43,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:58:46,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:51,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:56,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:58:58,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:58:58,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:58:58,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:02,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:07,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:09,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:59:12,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:13,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:14,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:14,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:18,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:18,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_65621df76009498b6856.png with policy now +2023-07-12 05:59:18,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_82f075e0330d5c29720e.png with policy now +2023-07-12 05:59:18,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_8760dbb0b29df7f09a82.png with policy now +2023-07-12 05:59:18,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 05:59:18,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1420_51314bed48a62a63179f.png with policy now +2023-07-12 05:59:18,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:18,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:18,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:18,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:18,841 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9b30smef-media/images/validation_1420_65621df76009498b6856.png +2023-07-12 05:59:18,887 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/75xlacfg-media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 05:59:18,905 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3khyp1e-media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 05:59:19,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:59:19,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_65621df76009498b6856.png +2023-07-12 05:59:19,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 05:59:19,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 05:59:19,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 05:59:19,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 05:59:19,448 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/10x4mewm-media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 05:59:19,733 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eucjy52z-media/images/validation_1420_65621df76009498b6856.png +2023-07-12 05:59:19,752 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6pk8oa2q-media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 05:59:19,762 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1xgxo4z-media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 05:59:19,786 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/50w60ak1-media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 05:59:23,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:25,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:28,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:28,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:28,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:28,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:29,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:31,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:34,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:37,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:37,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:37,368 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:37,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:37,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:59:37,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:39,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 05:59:40,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:43,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:44,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:44,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:44,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 05:59:46,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:46,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:46,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:46,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:46,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:49,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 05:59:49,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:52,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:55,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 05:59:58,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 05:59:58,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 05:59:58,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 05:59:58,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 05:59:58,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 05:59:59,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 05:59:59,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 05:59:59,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:01,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:01,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:03,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:07,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:09,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:00:09,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:09,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:09,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:09,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:09,512 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:13,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:13,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:14,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:00:14,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:14,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:15,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:19,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:19,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:19,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:19,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:19,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:20,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:25,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:25,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:25,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:29,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:00:29,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:29,246 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:29,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:29,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:29,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:29,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:30,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:31,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:31,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:35,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:37,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:39,300 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:00:39,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:39,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:39,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:39,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:41,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:43,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:43,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:44,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:44,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:00:47,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:49,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:50,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:50,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:50,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:50,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:52,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:54,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:00:54,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:00:54,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:00:54,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:00:55,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:00:55,690 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:00:58,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:00:59,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:00:59,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:01,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:01:04,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:09,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:09,300 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:01:14,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:14,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:14,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:14,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:19,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:24,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:29,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:29,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:29,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:30,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:35,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:39,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:01:41,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:44,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:44,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:44,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:01:46,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:49,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,246 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_a5e480b9cae1eda40b25.png with policy now +2023-07-12 06:01:49,255 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_6fd754942b22d95c6796.png with policy now +2023-07-12 06:01:49,266 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_bb8a00b843066df3ae5d.png with policy now +2023-07-12 06:01:49,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:01:49,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1430_acb50883d3668759985a.png with policy now +2023-07-12 06:01:49,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:01:49,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:01:49,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:01:49,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:01:49,541 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fq0jam1h-media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 06:01:49,556 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1phq2vjm-media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 06:01:49,590 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5jidxzj7-media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 06:01:49,599 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fb50ghw2-media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 06:01:49,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:01:49,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 06:01:49,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 06:01:49,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 06:01:49,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 06:01:49,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:01:50,301 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bb1zic44-media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 06:01:50,311 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l6a2yr5w-media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 06:01:50,322 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r5mektti-media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 06:01:50,324 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lug8f6id-media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 06:01:52,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:55,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:01:57,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:01:58,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:01:59,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:01:59,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:01:59,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:01,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:03,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:08,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:08,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:08,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:08,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:08,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:09,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:09,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:02:14,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:14,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:14,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:14,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:02:14,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:02:14,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:17,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:17,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:17,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:17,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:20,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:20,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:20,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:25,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:26,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:28,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:28,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:28,046 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:28,048 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:29,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:02:29,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:02:29,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:31,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:32,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:32,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:36,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:38,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:38,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:38,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:38,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:38,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:39,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:02:41,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:44,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:44,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:45,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:02:45,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:02:45,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:02:46,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:48,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:48,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:48,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:48,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:02:50,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:02:50,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:51,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:56,217 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:02:56,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:02:58,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:02:58,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:02:58,718 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:02:58,720 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:00,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:00,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:00,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:02,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:02,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:02,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:07,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:08,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:09,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:03:09,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:03:09,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:03:09,052 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:09,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:03:12,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:14,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:14,249 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:15,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:15,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:15,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:18,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:18,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:03:18,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:03:18,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:03:18,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:20,260 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:20,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:23,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:23,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:03:23,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:03:23,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:03:23,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:03:26,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:03:26,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:03:29,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:30,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:30,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:30,936 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:34,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:39,305 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:03:40,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:45,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:45,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:03:46,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:03:46,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:03:51,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:03:56,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:00,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:00,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:00,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:02,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:07,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:09,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:04:12,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:16,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:16,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:16,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:18,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:18,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_ff68423dd6be58cda77c.png with policy now +2023-07-12 06:04:18,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,290 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_7456d4bbc7e1692f0395.png with policy now +2023-07-12 06:04:18,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_2d1c628ad6cab3a92695.png with policy now +2023-07-12 06:04:18,312 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:04:18,312 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1440_84ea7cea928542cee5c3.png with policy now +2023-07-12 06:04:18,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:18,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:18,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:18,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:04:18,568 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s000v0hj-media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 06:04:18,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hauo85gy-media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 06:04:18,599 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3wbj8kwd-media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 06:04:18,612 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1cmlb1rf-media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 06:04:20,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:04:20,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 06:04:20,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 06:04:20,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 06:04:20,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 06:04:20,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:04:20,816 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ojkoaewj-media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 06:04:20,836 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/16abjk25-media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 06:04:20,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v1xy8c1k-media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 06:04:20,879 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ryac08n9-media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 06:04:23,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:26,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:28,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:29,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:31,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:31,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:31,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:32,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:34,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:37,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:37,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:37,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:37,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:04:38,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:04:38,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:39,306 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:04:40,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:44,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:46,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:47,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:04:47,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:04:47,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:04:47,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:47,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:47,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:47,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:04:50,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:04:50,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:51,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:56,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:04:56,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:04:58,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:04:58,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:04:58,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:04:58,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:02,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:02,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:02,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:02,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:02,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:02,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:07,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:08,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:09,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:09,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:09,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:09,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:09,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:05:13,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:14,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:14,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:18,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:18,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:18,072 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:18,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:18,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:18,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:18,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:18,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:20,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:20,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:23,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:26,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:28,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:28,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:28,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:28,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:29,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:32,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:32,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:33,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:33,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:33,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:34,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:38,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:39,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:39,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:39,244 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:39,246 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:39,309 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:05:40,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:44,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:44,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:45,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:48,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:05:48,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:05:48,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:05:49,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:49,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:49,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:49,146 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:50,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:50,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:05:51,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:53,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:05:53,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:05:53,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:05:53,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:05:56,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:05:56,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:05:56,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:06:01,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:03,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:07,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:09,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:06:13,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:18,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:18,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:18,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:18,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:23,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:28,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:33,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:33,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:33,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:34,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:39,311 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:06:40,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:45,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:48,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:06:48,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:06:48,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:06:48,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_6eeb827ca7d660925ab1.png with policy now +2023-07-12 06:06:48,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_1c96b81934ff3d0e0e7a.png with policy now +2023-07-12 06:06:48,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_f51ce3f90deb8ec9156b.png with policy now +2023-07-12 06:06:48,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:06:48,684 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1450_4fbcb05acbe8b16ae524.png with policy now +2023-07-12 06:06:48,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:06:48,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:06:48,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:06:48,727 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:06:48,965 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/feae43rl-media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 06:06:48,978 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t95a36dv-media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 06:06:48,992 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zur88039-media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 06:06:49,059 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d8glq9b5-media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 06:06:50,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:51,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:06:51,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 06:06:51,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 06:06:51,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 06:06:51,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 06:06:51,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:06:51,421 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2dk3b1ob-media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 06:06:51,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c3sbk92l-media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 06:06:51,455 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4r8erk3-media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 06:06:51,480 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/st813tzh-media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 06:06:56,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:06:57,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:06:58,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:01,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:03,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:03,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:08,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:08,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:09,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:09,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:09,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:09,477 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:09,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:07:15,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:15,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:15,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:18,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:18,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:18,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:18,184 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:18,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:18,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:18,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:20,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:21,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:21,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:25,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:27,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:28,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:28,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:28,299 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:28,301 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:30,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:33,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:33,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:33,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:36,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:38,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:38,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:38,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:38,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:39,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:39,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:39,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:07:42,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:45,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:48,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:07:48,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:07:48,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:07:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:49,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:49,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:49,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:07:51,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:07:51,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:53,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:57,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:07:58,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:07:59,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:07:59,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:07:59,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:07:59,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:03,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:03,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:03,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:03,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:03,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:03,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:09,315 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:08:09,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:09,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:09,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:08:09,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:08:09,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:08:09,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:14,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:15,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:15,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:18,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:18,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:18,902 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:19,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:08:19,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:08:19,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:08:19,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:20,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:21,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:21,374 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:24,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:08:24,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:08:24,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:08:24,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:08:25,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:27,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:08:27,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:08:30,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:34,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:34,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:34,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:36,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:39,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:08:41,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:46,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:49,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:08:49,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:08:49,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:08:52,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:08:57,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:02,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:04,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:04,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:04,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:07,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:09,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:09:12,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:17,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:18,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,790 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_fc2c6a80029568453397.png with policy now +2023-07-12 06:09:18,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_e0e86f6d8760c4a8cb7e.png with policy now +2023-07-12 06:09:18,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_bafecf6d5c7a9080d983.png with policy now +2023-07-12 06:09:18,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:09:18,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1460_688928104b18c14855ba.png with policy now +2023-07-12 06:09:18,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:18,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:18,890 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:18,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:09:19,116 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d6d4yzq2-media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 06:09:19,123 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ixw92z9-media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 06:09:19,137 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0b8nxsht-media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 06:09:19,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:19,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:19,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:19,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x4k7nj3a-media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 06:09:21,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:09:21,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 06:09:21,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 06:09:21,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 06:09:21,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 06:09:21,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:09:21,979 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ttix2yx5-media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 06:09:21,984 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3fedqpvk-media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 06:09:22,036 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0on5etyj-media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 06:09:22,046 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ollxezcs-media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 06:09:22,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:27,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:27,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:27,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:33,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:33,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:34,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:34,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:34,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:37,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:37,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:37,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:37,588 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:09:38,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:39,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:09:39,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:09:39,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:43,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:45,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:48,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:48,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:48,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:48,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:09:49,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:49,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:09:49,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:09:49,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:09:51,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:09:51,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:54,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:09:57,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:09:58,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:09:58,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:09:58,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:09:58,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:00,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:03,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:03,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:04,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:04,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:04,313 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:05,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:09,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:10:09,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:09,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:09,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:09,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:09,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:09,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:10,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:15,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:16,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:19,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:19,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:19,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:19,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:19,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:19,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:19,379 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:21,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:21,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:21,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:26,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:27,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:29,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:29,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:29,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:29,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:32,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:33,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:33,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:34,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:34,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:34,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:37,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:39,319 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:10:39,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:39,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:39,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:39,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:39,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:39,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:43,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:45,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:49,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:49,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:49,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:49,054 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:49,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:49,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:10:49,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:10:49,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:10:51,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:51,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:53,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:10:53,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:10:53,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:10:53,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:10:54,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:10:57,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:10:57,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:10:59,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:04,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:11:04,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:04,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:05,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:09,321 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:11:11,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:16,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:19,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:11:19,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:19,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:21,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:26,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:31,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:34,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:34,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:11:34,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:37,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:39,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:11:43,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:48,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_70617e686456e8dfd42a.png with policy now +2023-07-12 06:11:48,313 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,313 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_724c86f275815432b7e4.png with policy now +2023-07-12 06:11:48,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,314 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_841f2b0f1c0c1e17dd9d.png with policy now +2023-07-12 06:11:48,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:11:48,314 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1470_d752e1cd90b5707c0547.png with policy now +2023-07-12 06:11:48,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:11:48,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:11:48,348 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:11:48,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:48,349 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:11:48,579 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c9eh6n8h-media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 06:11:48,627 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hskj0hlt-media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 06:11:48,628 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hqjfwq55-media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 06:11:48,657 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ccgqw1h-media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 06:11:49,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:11:49,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:11:52,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:11:52,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 06:11:52,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 06:11:52,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 06:11:52,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 06:11:52,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:11:52,546 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pvv95184-media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 06:11:52,561 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hbc8avim-media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 06:11:52,633 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tzxznh8a-media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 06:11:52,649 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7swltc99-media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 06:11:53,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:11:57,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:11:58,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:11:58,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:04,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:04,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:04,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:04,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:07,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:07,331 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:07,331 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:07,333 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:09,323 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:12:10,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:10,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:10,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:16,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:16,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:16,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:16,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:16,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:16,998 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:19,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:19,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:12:19,447 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:21,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:22,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:22,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:26,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:27,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:27,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:27,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:27,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:28,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:28,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:32,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:34,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:35,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:12:35,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:35,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:37,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:38,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:38,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:38,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:38,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:39,325 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:12:40,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:40,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:43,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:46,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:48,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:49,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:49,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:49,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:49,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:50,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:12:50,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:12:50,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:12:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:12:52,462 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:54,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:12:58,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:12:58,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:12:58,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:12:58,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:12:58,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:12:59,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:04,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:04,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:05,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:05,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:05,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:05,954 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:08,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:13:08,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:13:08,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:13:08,859 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:13:09,325 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:13:10,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:10,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:10,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:15,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:16,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:19,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:13:19,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:13:19,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:13:19,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:13:21,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:21,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:21,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:21,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:22,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:22,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:23,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:13:23,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:13:23,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:13:23,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:13:26,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:13:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:13:31,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:36,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:36,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:36,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:37,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:39,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:13:42,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:47,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:51,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:13:51,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:13:51,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:13:53,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:13:58,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:03,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:06,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:06,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:06,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:08,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:09,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:14:13,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:18,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:18,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_a4d38678421736956450.png with policy now +2023-07-12 06:14:18,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_22921140211d309efc15.png with policy now +2023-07-12 06:14:18,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_577a6eadd1895fbf8edd.png with policy now +2023-07-12 06:14:18,617 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:14:18,617 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1480_17a3ddb8f942d0a4a42c.png with policy now +2023-07-12 06:14:18,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:18,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:18,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:18,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:14:18,891 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zvenh6eb-media/images/validation_1480_22921140211d309efc15.png +2023-07-12 06:14:18,927 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8uejm277-media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 06:14:18,933 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fhw63gwo-media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 06:14:18,936 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qlkf1ued-media/images/validation_1480_a4d38678421736956450.png +2023-07-12 06:14:21,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:21,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:21,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:22,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:14:22,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:22,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_a4d38678421736956450.png +2023-07-12 06:14:22,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 06:14:22,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_22921140211d309efc15.png +2023-07-12 06:14:22,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 06:14:22,765 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:14:23,152 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5i48pp2m-media/images/validation_1480_22921140211d309efc15.png +2023-07-12 06:14:23,161 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/579nxgn4-media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 06:14:23,162 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a26fke29-media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 06:14:23,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:23,496 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/93wdtvuc-media/images/validation_1480_a4d38678421736956450.png +2023-07-12 06:14:28,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:28,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:28,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:33,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:34,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:36,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:36,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:36,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:38,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:38,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:38,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:38,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:38,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:14:39,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:14:40,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:14:40,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:44,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:46,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:48,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:48,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:48,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:48,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:14:50,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:51,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:14:51,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:14:51,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:14:52,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:14:52,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:56,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:14:58,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:14:59,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:14:59,070 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:14:59,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:14:59,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:01,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:05,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:05,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:06,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:06,134 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:06,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:08,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:08,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:08,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:08,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:09,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:15:11,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:11,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:11,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:17,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:17,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:19,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:19,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:19,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:19,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:21,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:15:21,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:21,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:22,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:23,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:23,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:28,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:29,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:30,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:30,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:30,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:30,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:33,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:35,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:35,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:37,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:15:37,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:37,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:38,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:39,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:15:40,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:40,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:40,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:40,961 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:41,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:41,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:44,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:47,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:50,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:50,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:50,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:50,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:50,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:52,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:15:52,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:15:52,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:15:53,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:53,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:15:55,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:15:55,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:15:55,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:15:55,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:15:55,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:15:59,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:15:59,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:16:00,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:05,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:07,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:07,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:07,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:09,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:16:11,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:16,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:21,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:22,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:22,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:22,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:26,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:31,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:36,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:37,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:37,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:37,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:39,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:16:42,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:47,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:49,751 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,751 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_0084cd73f87c0d0b983c.png with policy now +2023-07-12 06:16:49,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,761 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_5443848005d4e6d0afaa.png with policy now +2023-07-12 06:16:49,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_4b4b982230746456caa0.png with policy now +2023-07-12 06:16:49,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:16:49,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1490_09e2e8132566f85bd196.png with policy now +2023-07-12 06:16:49,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:16:49,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:16:49,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:16:49,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:16:50,046 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8pjhc7sa-media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 06:16:50,062 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c2euj2gi-media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 06:16:50,087 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ouaf2f4-media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 06:16:50,088 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u2tkuufp-media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 06:16:52,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:52,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:16:52,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:16:52,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:16:53,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:16:53,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:16:53,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 06:16:53,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 06:16:53,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 06:16:53,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 06:16:53,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:16:53,815 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vcd954ma-media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 06:16:53,829 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sryu0cre-media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 06:16:53,832 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v9cb45fr-media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 06:16:53,842 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y0lcrnki-media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 06:16:58,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:16:59,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:00,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:03,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:06,495 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:07,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:07,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:07,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:08,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:09,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:17:10,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:10,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:10,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:10,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:13,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:13,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:14,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:20,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:20,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:20,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:20,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:20,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:20,887 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:22,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:22,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:22,818 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:25,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:27,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:27,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:31,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:31,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:31,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:31,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:31,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:34,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:34,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:36,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:38,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:38,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:38,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:39,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:17:41,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:41,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:41,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:41,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:41,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:41,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:42,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:48,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:48,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:51,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:17:51,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:17:51,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:17:51,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:17:53,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:17:53,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:17:53,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:17:53,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:17:55,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:17:55,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:17:58,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:02,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:02,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:02,082 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:02,084 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:02,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:02,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:04,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:08,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:08,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:08,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:09,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:09,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:18:09,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:12,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:12,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:12,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:12,254 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:14,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:16,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:16,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:19,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:22,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:22,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:22,743 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:22,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:23,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:23,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:24,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:24,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:24,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:24,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:27,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:18:27,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:18:27,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:18:27,176 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:18:29,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:30,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:18:30,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:18:34,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:39,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:18:39,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:39,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:39,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:40,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:45,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:50,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:18:54,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:18:54,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:18:54,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:18:55,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:00,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:05,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:09,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:19:09,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:09,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:09,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:10,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:15,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:20,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:21,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_c6e34c37f93334a75319.png with policy now +2023-07-12 06:19:21,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_de9f5fe5746030c6a74f.png with policy now +2023-07-12 06:19:21,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_cfb49806a7152804aa8b.png with policy now +2023-07-12 06:19:21,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:19:21,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1500_9a03a9d55b0a9f41b234.png with policy now +2023-07-12 06:19:22,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:22,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:19:22,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:19:22,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:19:22,301 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/euqe6m5o-media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 06:19:22,327 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/53lw2ti7-media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 06:19:22,344 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n1qw7ry6-media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 06:19:22,362 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qwq9mtlj-media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 06:19:24,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:24,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:24,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:26,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:27,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:19:27,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 06:19:27,053 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 06:19:27,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 06:19:27,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 06:19:27,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:19:27,482 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lb1rqtn9-media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 06:19:27,492 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7fcsfs6r-media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 06:19:27,503 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8jwnbc9d-media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 06:19:27,514 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5goed0qy-media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 06:19:32,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:32,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:34,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:37,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:39,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:19:39,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:39,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:39,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:41,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:41,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:41,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:19:41,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:19:41,253 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:19:43,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:48,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:19:48,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:48,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:51,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:19:51,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:19:51,420 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:19:51,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:19:53,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:19:54,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:19:54,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:19:54,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:19:55,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:19:55,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:19:58,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:02,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:02,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:02,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:02,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:02,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:02,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:04,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:09,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:09,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:20:09,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:09,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:09,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:09,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:11,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:11,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:11,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:12,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:14,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:16,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:16,144 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:20,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:22,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:22,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:22,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:22,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:23,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:23,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:24,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:24,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:24,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:25,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:30,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:30,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:32,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:32,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:32,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:32,374 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:36,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:37,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:37,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:39,343 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:20:39,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:39,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:39,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:41,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:42,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:42,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:42,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:42,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:44,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:44,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:47,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:51,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:20:52,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:52,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:52,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:52,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:52,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:54,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:20:54,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:20:54,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:20:57,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:20:57,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:20:57,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:20:57,587 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:20:57,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:20:58,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:20:58,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:21:03,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:05,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:21:08,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:09,344 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:21:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:09,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:09,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:21:13,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:18,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:23,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:24,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:24,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:24,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:21:29,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:34,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:39,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:39,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:21:40,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:40,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:40,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:21:44,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:49,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:51,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,904 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_be748917843477368009.png with policy now +2023-07-12 06:21:51,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_ec02c5d5a2c3f65267f6.png with policy now +2023-07-12 06:21:51,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_d4a2e70a118507b473e4.png with policy now +2023-07-12 06:21:51,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:21:51,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1510_11ba5556c102236aa637.png with policy now +2023-07-12 06:21:51,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:21:51,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:21:51,984 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:21:51,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:21:52,202 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ca20jn28-media/images/validation_1510_be748917843477368009.png +2023-07-12 06:21:52,225 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qdf8ditg-media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 06:21:52,232 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n9yjifrb-media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 06:21:52,249 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iyqwlfq5-media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 06:21:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:21:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 06:21:54,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 06:21:54,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 06:21:54,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_be748917843477368009.png +2023-07-12 06:21:54,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:21:54,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:21:54,908 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kokerfdz-media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 06:21:54,918 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/68uy5ee7-media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 06:21:54,924 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2z7g3bep-media/images/validation_1510_be748917843477368009.png +2023-07-12 06:21:54,927 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blwj50d1-media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 06:21:55,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:21:55,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:21:55,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:00,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:01,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:01,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:05,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:08,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:09,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:22:10,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:10,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:10,754 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:11,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:11,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:11,321 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:11,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:11,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:15,608 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:15,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:17,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:21,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:21,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:21,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:21,816 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:22,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:26,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:26,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:26,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:28,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:29,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:32,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:32,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:32,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:32,659 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:33,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:36,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:36,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:39,348 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:22:39,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:41,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:41,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:41,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:42,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:42,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:42,137 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:42,139 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:43,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:43,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:44,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:49,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:50,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:22:52,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:22:52,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:22:52,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:22:52,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:22:54,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:22:57,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:22:57,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:22:57,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:22:57,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:22:57,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:00,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:02,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:02,767 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:02,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:02,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:04,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:04,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:05,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:09,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:23:10,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:11,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:12,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:12,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:12,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:12,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:12,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:12,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:12,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:15,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:18,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:18,854 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:21,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:22,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:22,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:22,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:22,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:25,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:25,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:26,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:27,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:23:27,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:23:27,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:23:27,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:23:27,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:27,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:27,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:31,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:33,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:23:33,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:23:36,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:39,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:23:42,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:42,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:42,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:42,620 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:47,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:52,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:23:57,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:23:57,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:23:57,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:23:58,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:03,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:08,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:09,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:24:12,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:24:12,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:12,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:24:13,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:18,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:21,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_3d2b9accd30d9956baeb.png with policy now +2023-07-12 06:24:21,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_634e550898bfbdb9ac9f.png with policy now +2023-07-12 06:24:21,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_790fbc2ddb08d6dd0d88.png with policy now +2023-07-12 06:24:21,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:24:21,395 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1520_5140a0409a56d2f968a2.png with policy now +2023-07-12 06:24:21,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:21,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:24:21,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:24:21,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:24:21,684 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/suof8yrt-media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 06:24:21,696 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjcv0qpf-media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 06:24:21,720 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cspxx5or-media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 06:24:21,830 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qavi2ft6-media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 06:24:22,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:24:22,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 06:24:22,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 06:24:22,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 06:24:22,208 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 06:24:22,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:24:22,570 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ranvhjui-media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 06:24:22,572 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vkrcq6yg-media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 06:24:22,575 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v7ddkiw4-media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 06:24:22,663 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tqt1nhuw-media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 06:24:24,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:27,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:27,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:24:27,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:24:29,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:29,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:35,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:36,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:39,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:24:40,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:41,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:41,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:24:41,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:24:41,053 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:24:42,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:42,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:24:43,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:24:43,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:46,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:50,258 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:50,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:24:50,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:24:50,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:24:50,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:24:51,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:56,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:24:57,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:24:57,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:24:57,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:24:57,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:00,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:00,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:00,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:00,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:01,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:04,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:04,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:06,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:09,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:25:11,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:11,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:11,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:11,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:11,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:11,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:12,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:13,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:13,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:13,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:18,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:18,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:20,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:20,976 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:20,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:20,978 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:23,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:25,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:25,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:28,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:28,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:28,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:29,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:31,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:31,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:31,051 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:31,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:32,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:32,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:35,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:39,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:25:39,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:40,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:41,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:41,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:41,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:41,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:43,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:43,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:25:45,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:46,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:46,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:51,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:52,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:52,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:52,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:52,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:53,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:25:53,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:25:56,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:25:57,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:25:57,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:25:57,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:25:57,123 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:25:58,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:25:58,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:25:58,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:00,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:26:00,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:26:02,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:07,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:09,358 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:26:13,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:13,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:13,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:26:13,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:18,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:23,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:28,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:28,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:28,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:33,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:38,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:39,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:26:43,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:26:43,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:43,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:43,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:48,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:51,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_c983d04e3b22e1d9100a.png with policy now +2023-07-12 06:26:51,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,565 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_91df6d540afc90aa229c.png with policy now +2023-07-12 06:26:51,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_7e2daf917c565f387c73.png with policy now +2023-07-12 06:26:51,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:26:51,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1530_6550992879b720b36137.png with policy now +2023-07-12 06:26:51,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:26:51,625 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:26:51,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:26:51,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:26:51,866 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8e4y9sja-media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 06:26:51,874 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i1uwyt11-media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 06:26:51,889 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/obqfolwo-media/images/validation_1530_6550992879b720b36137.png +2023-07-12 06:26:51,902 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xug1yaoa-media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 06:26:53,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:26:56,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:26:56,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:26:56,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 06:26:56,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_6550992879b720b36137.png +2023-07-12 06:26:56,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 06:26:56,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 06:26:56,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:26:57,097 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uyv2o5hh-media/images/validation_1530_6550992879b720b36137.png +2023-07-12 06:26:57,104 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5bcoyjf8-media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 06:26:57,110 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/82y3z0rh-media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 06:26:57,110 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pflkmxd2-media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 06:26:58,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:26:58,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:26:58,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:26:58,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:01,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:03,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:03,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:09,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:09,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:27:10,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:11,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:11,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:11,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:11,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:13,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:13,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:13,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:27:14,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:17,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:17,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:20,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:21,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:21,853 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:21,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:21,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:24,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:24,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:26,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:28,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:28,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:28,800 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:27:31,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:31,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:31,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:31,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:31,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:36,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:38,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:39,364 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:27:41,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:41,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:41,843 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:41,843 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:41,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:43,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:43,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:43,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:27:45,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:45,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:47,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:51,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:27:51,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:27:51,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:27:51,849 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:27:52,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:52,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:27:52,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:27:58,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:27:58,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:27:58,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:27:58,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:00,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:01,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:01,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:01,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:01,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:03,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:07,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:28:07,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:09,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:09,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:28:12,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:12,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:12,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:12,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:14,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:28:14,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:14,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:14,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:14,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:19,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:21,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:22,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:22,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:22,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:22,483 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:25,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:27,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:28:27,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:28:27,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:28:27,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:28:28,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:28:28,064 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:29,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:29,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:29,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:30,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:35,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:28:35,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:39,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:28:40,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:44,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:44,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:44,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:28:46,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:51,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:56,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:28:59,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:28:59,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:28:59,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:01,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:06,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:09,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:29:11,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:14,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:14,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:14,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:17,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:21,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:21,999 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_5f4cc7e816317d29edbf.png with policy now +2023-07-12 06:29:22,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:22,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_a3ac4868653991969a01.png with policy now +2023-07-12 06:29:22,031 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:22,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_d7dea15105ec789e1d21.png with policy now +2023-07-12 06:29:22,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:29:22,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1540_cd95bda74fc7ee9c42fd.png with policy now +2023-07-12 06:29:22,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:22,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:29:22,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:29:22,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:29:22,339 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q6v3lzb6-media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 06:29:22,351 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mgjiihc2-media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 06:29:22,353 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lcbbsnbp-media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 06:29:22,362 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/opyq82er-media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 06:29:23,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:24,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:29:24,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 06:29:24,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 06:29:24,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 06:29:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 06:29:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:29:24,626 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jv1jesqd-media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 06:29:24,640 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mmui4izt-media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 06:29:24,650 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z0ubowlj-media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 06:29:24,683 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/75w12nqj-media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 06:29:28,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:29,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:29,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:29,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:31,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:31,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:33,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:38,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:39,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:39,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:29:41,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:41,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:29:41,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:29:41,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:29:44,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:44,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:44,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:29:44,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:45,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:29:45,338 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:50,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:50,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:29:50,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:29:50,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:29:50,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:29:52,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:29:52,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:55,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:29:59,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:29:59,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:29:59,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:29:59,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:00,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:01,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:01,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:01,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:01,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:06,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:06,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:06,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:09,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:30:11,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:12,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:12,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:12,124 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:12,126 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:13,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:13,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:14,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:30:14,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:30:14,432 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:16,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:20,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:22,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:22,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:22,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:22,963 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:22,965 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:27,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:27,552 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:27,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:30,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:30:30,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:30:30,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:32,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:33,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:33,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:33,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:33,545 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:34,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:34,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:38,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:39,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:30:41,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:43,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:43,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:43,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:43,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:43,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:45,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:30:45,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:30:45,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:30:48,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:48,592 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:48,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:53,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:53,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:30:53,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:53,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:53,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:55,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:30:55,605 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:30:58,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:30:58,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:30:58,626 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:30:58,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:30:59,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:00,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:00,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:00,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:02,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:31:02,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:31:04,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:09,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:31:10,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:15,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:15,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:15,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:15,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:20,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:25,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:30,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:30,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:30,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:31,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:36,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:39,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:31:42,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:45,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:31:45,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:31:45,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:31:47,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:52,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:53,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_011beff0267deaa0300b.png with policy now +2023-07-12 06:31:53,462 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,462 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_8efbafcfdf5d8df07263.png with policy now +2023-07-12 06:31:53,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_1b7f50867d7f974e4323.png with policy now +2023-07-12 06:31:53,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:31:53,520 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1550_bd10f02e7a1744e7844e.png with policy now +2023-07-12 06:31:53,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:31:53,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:31:53,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:31:53,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:31:53,797 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/217dm9zw-media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 06:31:53,801 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dlz8umik-media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 06:31:53,811 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7aug2hw-media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 06:31:53,833 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ytkm9a1t-media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 06:31:57,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:31:58,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:31:58,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:31:58,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 06:31:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 06:31:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 06:31:58,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 06:31:58,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:31:59,189 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tj914um9-media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 06:31:59,201 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b50vkfq7-media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 06:31:59,210 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cfjs8hhj-media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 06:31:59,212 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bt1whog2-media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 06:32:00,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:00,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:00,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:03,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:03,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:05,871 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:09,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:09,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:32:12,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:12,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:12,828 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:12,830 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:12,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:12,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:14,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:15,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:15,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:15,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:20,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:20,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:22,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:22,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:22,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:22,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:26,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:27,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:27,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:30,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:30,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:30,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:32,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:33,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:33,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:33,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:33,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:34,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:34,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:37,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:39,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:32:41,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:43,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:43,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:43,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:43,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:43,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:46,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:32:46,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:32:46,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:32:48,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:48,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:48,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:53,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:32:54,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:32:54,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:32:54,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:32:54,413 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:32:55,079 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:32:55,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:32:59,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:01,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:01,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:01,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:02,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:04,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:05,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:05,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:05,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:05,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:09,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:09,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:09,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:33:10,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:15,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:15,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:15,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:15,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:15,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:16,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:16,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:16,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:16,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:16,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:20,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:23,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:25,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:25,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:25,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:25,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:26,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:30,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:30,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:30,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:33:30,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:33:30,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:33:30,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:33:31,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:31,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:31,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:32,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:37,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:33:37,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:33:37,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:39,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:33:43,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:46,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:33:46,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:33:46,438 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:33:48,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:53,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:33:58,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:01,438 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:04,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:09,379 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:34:10,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:15,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:16,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:16,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:16,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:20,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:25,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_b236feda2a099cb72a65.png with policy now +2023-07-12 06:34:25,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,208 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_503b3c4a7c702e3bde04.png with policy now +2023-07-12 06:34:25,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_fb2e82a82bfa5264522f.png with policy now +2023-07-12 06:34:25,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:34:25,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1560_137a9e1a20d097ef6ead.png with policy now +2023-07-12 06:34:25,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:25,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:34:25,303 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:34:25,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:34:25,508 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tglkrqwx-media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 06:34:25,515 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/096qyqci-media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 06:34:25,571 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1uoeijv-media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 06:34:25,605 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cph02j2v-media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 06:34:26,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:26,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:34:26,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 06:34:26,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 06:34:26,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 06:34:26,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 06:34:26,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:34:26,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g0cffeqo-media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 06:34:26,858 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0jsdcsqm-media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 06:34:26,874 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zxto74ax-media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 06:34:26,892 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4p110s8g-media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 06:34:31,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:31,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:31,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:31,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:33,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:35,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:37,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:39,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:34:40,546 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:42,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:44,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:44,942 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:34:44,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:34:44,944 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:34:47,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:34:47,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:34:47,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:34:47,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:34:47,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:47,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:52,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:34:54,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:34:55,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:34:55,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:34:55,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:34:55,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:34:58,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:01,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:01,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:03,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:03,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:03,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:04,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:06,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:06,315 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:06,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:06,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:08,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:08,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:09,381 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:35:10,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:15,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:16,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:16,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:16,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:16,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:16,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:18,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:18,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:18,883 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:21,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:22,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:26,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:27,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:27,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:27,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:27,546 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:29,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:29,770 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:32,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:34,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:34,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:34,321 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:36,783 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:37,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:37,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:37,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:37,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:37,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:39,381 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:35:43,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:43,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:47,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:47,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:47,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:47,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:49,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:49,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:35:49,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:35:49,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:35:50,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:50,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:54,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:35:57,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:35:57,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:35:57,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:35:57,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:35:57,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:35:57,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:35:59,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:01,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:36:01,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:36:01,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:36:01,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:36:04,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:04,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:04,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:04,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:36:04,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:36:05,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:09,383 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:36:11,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:16,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:19,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:19,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:19,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:21,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:26,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:31,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:34,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:34,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:34,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:36,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:39,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:36:42,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:47,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:49,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:36:49,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:36:49,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:36:52,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:36:56,488 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,489 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_fb05536233b9d9384d99.png with policy now +2023-07-12 06:36:56,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_8b73b1d4908d57e0f731.png with policy now +2023-07-12 06:36:56,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_c996b3e0e81ceb767ce9.png with policy now +2023-07-12 06:36:56,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:36:56,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1570_cdd5f325de677a2f831c.png with policy now +2023-07-12 06:36:56,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:36:56,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:36:56,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:36:56,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:36:56,816 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9919zxeg-media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 06:36:56,823 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ykefeptl-media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 06:36:56,826 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ztuz0135-media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 06:36:56,831 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j6po7pvf-media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 06:36:57,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:01,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:01,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:01,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 06:37:01,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 06:37:01,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 06:37:01,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 06:37:01,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:37:01,413 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oanc6q8n-media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 06:37:01,423 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/psglyj6a-media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 06:37:01,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2593vaiy-media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 06:37:01,449 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1kkvcqv8-media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 06:37:03,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:04,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:04,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:04,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:06,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:08,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:08,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:09,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:37:14,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:15,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:16,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:16,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:16,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:16,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:20,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:20,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:20,137 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:20,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:22,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:22,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:25,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:25,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:25,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:25,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:25,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:29,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:29,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:31,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:35,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:35,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:35,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:36,138 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:36,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:36,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:36,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:36,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:36,190 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:39,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:37:42,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:43,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:43,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:46,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:46,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:46,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:46,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:47,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:50,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:50,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:50,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:37:50,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:37:50,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:37:52,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:37:55,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:37:55,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:37:55,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:37:55,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:37:57,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:37:57,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:37:57,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:02,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:04,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:05,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:38:05,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:05,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:05,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:05,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:05,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:05,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:08,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:09,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:38:11,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:38:11,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:13,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:15,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:15,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:15,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:15,464 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:18,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:38:18,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:19,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:20,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:38:20,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:20,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:24,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:25,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:26,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:26,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:26,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:26,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:30,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:31,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:38:31,440 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:38:31,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:38:31,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:38:32,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:38:32,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:36,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:36,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:38:36,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:36,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:39,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:38:39,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:38:41,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:46,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:51,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:38:51,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:38:52,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:38:57,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:02,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:06,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:06,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:06,325 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:07,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:09,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:39:12,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:17,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:21,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:21,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:21,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:23,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:25,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_934a30ce9258abe3af2d.png with policy now +2023-07-12 06:39:25,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_d7d4dbe10d69395e40ca.png with policy now +2023-07-12 06:39:25,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,673 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_37f9cd55db6f0502d41d.png with policy now +2023-07-12 06:39:25,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:39:25,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1580_cb784df1b13ebac5484a.png with policy now +2023-07-12 06:39:25,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:25,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:39:25,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:39:25,727 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:39:25,987 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgqjc1g2-media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 06:39:26,009 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ny3skwbi-media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 06:39:26,015 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/earbd7h5-media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 06:39:26,033 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wu6z9f2x-media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 06:39:28,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:28,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:39:28,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 06:39:28,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 06:39:28,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 06:39:28,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 06:39:28,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:39:29,010 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fnnp3zp1-media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 06:39:29,032 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xhvjah59-media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 06:39:29,032 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/os4jvlpk-media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 06:39:29,053 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g9uoglw8-media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 06:39:34,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:35,640 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:39:35,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:36,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:36,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:36,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:39,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:39:39,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:42,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:39:44,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:44,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:44,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:39:44,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:39:44,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:39:49,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:39:49,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:39:49,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:51,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:39:51,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:39:51,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:39:54,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:39:54,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:39:54,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:39:54,173 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:39:55,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:39:56,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:39:56,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:00,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:03,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:04,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:04,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:04,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:04,261 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:06,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:06,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:06,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:06,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:09,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:40:10,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:10,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:11,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:14,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:14,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:14,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:14,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:16,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:17,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:17,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:21,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:21,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:21,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:21,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:24,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:25,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:25,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:25,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:25,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:27,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:31,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:31,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:32,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:35,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:35,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:35,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:35,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:37,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:37,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:37,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:38,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:38,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:38,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:39,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:40:43,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:45,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:45,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:45,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:45,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:45,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:48,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:52,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:52,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:40:53,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:40:53,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:40:53,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:40:53,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:56,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:40:56,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:40:56,704 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:40:56,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:40:58,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:40:59,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:40:59,928 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:41:01,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:41:01,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:41:01,531 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:41:01,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:41:04,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:06,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:41:06,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:41:08,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:08,098 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:08,099 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:09,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:09,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:41:14,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:19,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:23,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:23,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:23,325 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:25,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:30,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:35,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:38,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:38,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:38,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:39,394 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:41:40,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:45,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:50,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:53,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:41:53,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:41:53,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:41:55,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:41:55,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:55,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_d7d78a7a562011ba37c0.png with policy now +2023-07-12 06:41:55,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:55,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_bada9f8f4e29cb81aa9e.png with policy now +2023-07-12 06:41:55,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:55,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_da4a4411337b18959a1d.png with policy now +2023-07-12 06:41:56,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:41:56,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1590_351faecc0b8489584c40.png with policy now +2023-07-12 06:41:56,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:41:56,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:41:56,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:41:56,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:41:56,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:41:56,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 06:41:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 06:41:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 06:41:56,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 06:41:56,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:41:56,276 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3gonoxnw-media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 06:41:56,304 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3l1qcaon-media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 06:41:56,309 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qgdltk7i-media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 06:41:56,395 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g21nu8lr-media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 06:41:56,794 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s4sujjm1-media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 06:41:56,912 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7k5lafm-media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 06:41:56,941 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l55i3t3e-media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 06:41:57,214 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8wd8s1c5-media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 06:42:01,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:03,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:06,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:06,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:08,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:08,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:08,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:09,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:42:10,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:11,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:16,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:16,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:16,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:16,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:16,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:17,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:17,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:22,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:23,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:23,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:23,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:24,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:26,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:26,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:26,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:26,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:27,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:31,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:31,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:33,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:36,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:36,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:36,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:36,639 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:38,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:38,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:38,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:39,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:39,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:39,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:39,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:42:44,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:45,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:47,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:47,274 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:47,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:47,276 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:49,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:52,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:52,262 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:42:54,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:42:54,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:42:54,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:42:55,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:42:57,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:42:57,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:42:57,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:42:57,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:42:59,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:42:59,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:00,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:06,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:06,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:07,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:07,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:07,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:07,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:09,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:43:09,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:09,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:09,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:11,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:13,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:13,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:17,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:17,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:17,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:17,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:17,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:20,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:20,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:23,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:24,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:24,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:24,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:26,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:26,767 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:26,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:26,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:27,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:27,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:28,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:31,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:43:31,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:43:31,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:43:31,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:43:34,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:34,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:43:34,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:43:39,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:39,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:43:39,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:39,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:39,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:44,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:49,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:43:54,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:43:54,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:43:54,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:43:55,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:00,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:05,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:09,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:44:09,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:09,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:09,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:10,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:15,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:20,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:24,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:24,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:24,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:25,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:26,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,517 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_a9454bcbee87eab5aea9.png with policy now +2023-07-12 06:44:26,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,526 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_9092d43f6e1c508c45c8.png with policy now +2023-07-12 06:44:26,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,536 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_214ff49c4cb987612b43.png with policy now +2023-07-12 06:44:26,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:44:26,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1600_aeea6b2da2c3bf58c7c3.png with policy now +2023-07-12 06:44:26,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:26,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:44:26,588 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:44:26,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:44:26,825 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ddiswf1u-media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 06:44:26,840 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vowqdifq-media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 06:44:26,857 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7dqhkmg-media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 06:44:26,891 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ym7mv0n-media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 06:44:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 06:44:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 06:44:30,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:44:31,059 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c9o3ytf5-media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 06:44:31,083 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kfg5k53t-media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 06:44:31,125 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/upcl89lk-media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 06:44:31,146 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dlgy8n3l-media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 06:44:31,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:36,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:36,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:37,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:44:39,402 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:44:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:39,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:41,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:44,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:44:46,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:46,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:46,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:44:46,537 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:44:46,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:44:51,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:44:51,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:44:51,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:54,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:44:54,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:44:54,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:44:56,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:44:56,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:44:56,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:44:56,628 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:44:57,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:44:58,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:44:58,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:02,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:05,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:06,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:06,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:06,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:06,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:07,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:09,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:45:09,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:09,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:09,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:12,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:12,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:12,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:17,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:17,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:17,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:17,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:18,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:19,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:19,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:23,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:25,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:25,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:25,441 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:26,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:28,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:28,513 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:28,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:28,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:29,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:33,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:34,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:39,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:39,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:39,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:39,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:39,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:45:40,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:40,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:40,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:41,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:41,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:41,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:46,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:48,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:49,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:49,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:49,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:49,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:45:51,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:55,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:45:55,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:45:56,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:45:56,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:45:56,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:45:57,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:45:58,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:45:58,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:45:58,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:45:58,616 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:46:02,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:46:02,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:46:03,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:03,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:46:03,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:46:03,200 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:46:03,202 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:46:08,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:09,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:46:09,137 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:46:09,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:46:11,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:11,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:11,348 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:14,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:19,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:24,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:26,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:26,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:26,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:29,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:34,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:39,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:46:40,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:41,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:41,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:41,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:45,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:50,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:55,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:46:56,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:46:56,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:46:56,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:46:57,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,838 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_1733cdf33e35398110c3.png with policy now +2023-07-12 06:46:57,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,847 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_b6d946329d7752479070.png with policy now +2023-07-12 06:46:57,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_f89b822e56af0bee16c1.png with policy now +2023-07-12 06:46:57,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:46:57,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1610_49a77c36b783c7c3533f.png with policy now +2023-07-12 06:46:57,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:46:57,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:46:57,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:46:57,911 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:46:58,171 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/afstmn9k-media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 06:46:58,186 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xl5pk6tt-media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 06:46:58,196 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/relcfh1z-media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 06:46:58,216 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mg9a4iul-media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 06:46:58,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:46:58,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 06:46:58,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 06:46:58,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 06:46:58,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 06:46:58,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:46:58,723 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cdkpx4pv-media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 06:46:58,738 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2ez19ru0-media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 06:46:58,755 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/at2ihzyc-media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 06:46:58,770 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fhvva6p6-media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 06:47:00,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:05,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:06,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:07,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:09,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:47:11,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:11,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:11,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:11,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:12,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:16,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:17,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:17,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:17,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:17,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:19,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:19,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:21,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:26,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:26,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:26,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:26,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:26,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:27,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:27,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:27,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:27,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:32,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:33,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:33,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:37,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:37,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:37,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:37,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:37,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:39,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:47:40,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:40,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:41,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:41,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:41,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:43,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:47,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:47,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:47,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:47,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:47,457 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:47:48,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:54,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:47:54,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:47:54,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:47:56,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:47:56,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:47:56,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:47:57,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:47:57,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:47:57,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:47:58,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:00,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:01,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:01,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:05,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:07,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:07,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:07,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:07,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:08,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:08,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:09,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:48:10,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:12,488 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:15,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:15,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:18,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:18,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:18,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:18,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:21,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:22,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:22,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:26,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:27,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:27,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:27,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:28,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:28,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:28,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:28,855 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:29,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:29,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:32,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:33,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:48:33,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:48:33,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:48:33,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:48:36,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:48:36,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:48:37,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:39,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:48:42,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:42,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:42,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:43,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:48,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:53,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:48:57,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:48:57,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:48:57,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:48:58,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:03,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:08,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:09,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:49:12,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:12,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:12,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:49:13,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:18,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:23,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:27,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_5f0c6d880a0a69fc8df8.png with policy now +2023-07-12 06:49:27,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_cab2ce99009b26517a5a.png with policy now +2023-07-12 06:49:27,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_5e34d10c6734761c103e.png with policy now +2023-07-12 06:49:27,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:49:27,526 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1620_0772971e314e9d07f59b.png with policy now +2023-07-12 06:49:27,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:27,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:49:27,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:49:27,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:49:27,849 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/db97lb2x-media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 06:49:27,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:27,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:27,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:49:27,874 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uz42jcku-media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 06:49:27,876 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b1l27awm-media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 06:49:27,879 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/67yfky8j-media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 06:49:29,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:32,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:49:32,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:32,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 06:49:32,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 06:49:32,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 06:49:32,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 06:49:32,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:49:33,537 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ii8z922h-media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 06:49:33,538 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rd8ky0z8-media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 06:49:33,589 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7p8hzmod-media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 06:49:33,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hzg29370-media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 06:49:34,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:37,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:39,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:49:39,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:40,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:43,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:43,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:43,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:49:45,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:46,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:46,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:49:46,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:49:46,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:49:46,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:49:46,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:51,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:53,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:49:56,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:49:56,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:49:56,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:49:56,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:49:56,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:49:58,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:49:58,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:49:58,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:00,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:00,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:02,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:06,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:06,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:06,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:06,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:07,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:07,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:08,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:09,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:50:13,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:13,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:14,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:16,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:16,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:16,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:16,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:18,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:22,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:22,010 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:23,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:27,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:27,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:27,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:27,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:28,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:28,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:28,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:29,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:29,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:29,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:35,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:36,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:38,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:38,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:38,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:38,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:39,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:50:40,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:43,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:43,195 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:43,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:43,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:43,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:50:45,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:48,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:48,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:48,562 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:48,564 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:50,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:50:50,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:50,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:56,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:50:57,223 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:50:58,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:50:58,365 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:50:58,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:50:58,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:50:58,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:50:58,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:50:58,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:01,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:02,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:51:02,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:51:02,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:51:02,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:51:04,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:51:04,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:51:07,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:09,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:51:11,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:51:13,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:13,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:13,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:13,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:18,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:23,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:28,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:28,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:28,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:28,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:33,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:38,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:39,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:51:43,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:43,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:43,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:44,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:49,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:54,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:51:57,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_72968a408eafe4615fe4.png with policy now +2023-07-12 06:51:57,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_de760783b75bdc640c99.png with policy now +2023-07-12 06:51:57,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_bcdd70acdf0c52445fdc.png with policy now +2023-07-12 06:51:57,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:51:57,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1630_661f798cc99bf53f5fc2.png with policy now +2023-07-12 06:51:57,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:51:57,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:51:57,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:51:57,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:51:57,797 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4exh7hav-media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 06:51:57,836 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/epiw2mfp-media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 06:51:57,868 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3300fubv-media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 06:51:57,872 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kju7u05q-media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 06:51:58,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:51:58,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:51:58,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:51:59,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:00,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:00,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 06:52:00,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 06:52:00,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 06:52:00,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 06:52:00,477 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:52:00,828 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ca3z86kw-media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 06:52:00,851 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3a41j9xi-media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 06:52:00,857 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fo4idme-media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 06:52:00,860 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r86dnv1g-media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 06:52:05,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:07,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:07,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:09,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:52:10,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:13,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:52:13,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:52:13,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:52:14,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:16,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:16,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:16,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:16,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:16,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:21,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:21,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:22,027 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:26,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:26,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:26,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:26,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:27,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:28,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:28,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:29,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:52:29,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:52:29,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:52:33,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:35,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:37,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:37,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:37,715 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:37,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:38,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:39,422 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:52:42,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:42,560 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:44,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:45,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:52:45,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:52:45,086 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:52:48,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:48,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:48,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:48,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:52:49,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:49,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:52:49,573 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:54,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:52:56,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:52:58,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:52:58,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:52:58,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:52:58,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:00,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:00,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:00,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:00,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:03,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:03,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:05,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:08,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:08,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:08,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:08,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:09,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:53:10,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:10,755 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:11,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:15,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:15,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:15,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:17,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:17,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:18,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:18,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:18,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:18,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:22,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:24,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:24,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:27,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:27,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:27,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:27,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:27,728 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:30,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:30,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:30,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:31,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:31,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:32,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:53:32,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:32,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:53:32,834 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:53:32,836 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:53:38,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:38,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:53:38,802 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:53:39,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:53:44,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:45,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:53:45,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:53:45,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:53:49,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:54,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:53:59,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:00,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:00,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:00,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:04,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:09,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:54:10,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:15,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:15,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:15,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:15,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:20,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:25,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:27,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_a26ecf3db23e2ccac681.png with policy now +2023-07-12 06:54:27,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_585881b9e0f6d40e0ec3.png with policy now +2023-07-12 06:54:27,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,574 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_68a5f846f45f697c2ead.png with policy now +2023-07-12 06:54:27,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:54:27,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1640_64d777e8b185c0851910.png with policy now +2023-07-12 06:54:27,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:27,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:54:27,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:54:27,611 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:54:27,848 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmaptz4j-media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 06:54:27,900 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5z5ilxw5-media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 06:54:27,903 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02974kkn-media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 06:54:27,909 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/twvu0oie-media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 06:54:28,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:54:28,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 06:54:28,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 06:54:28,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 06:54:28,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 06:54:28,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:54:28,375 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/63lurbvm-media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 06:54:28,396 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dt3p7hp1-media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 06:54:28,409 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0v0iy29p-media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 06:54:28,410 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dccb0g4a-media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 06:54:30,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:30,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:30,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:30,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:35,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:36,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:37,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:39,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:54:41,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:42,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:46,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:54:46,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:54:46,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:54:46,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:46,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:46,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:54:46,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:54:46,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:54:49,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:54:49,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:51,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:56,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:54:57,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:54:57,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:54:57,220 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:54:57,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:54:57,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:01,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:01,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:01,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:02,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:03,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:03,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:07,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:07,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:07,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:07,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:08,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:09,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:55:10,122 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:10,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:13,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:17,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:17,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:17,100 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:17,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:17,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:17,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:17,668 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:17,670 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:18,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:23,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:24,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:24,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:27,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:27,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:27,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:27,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:29,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:31,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:31,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:32,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:34,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:37,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:37,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:37,961 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:37,963 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:38,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:38,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:39,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:55:40,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:45,354 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:45,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:47,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:55:47,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:55:47,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:55:48,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:48,322 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:48,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:48,324 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:50,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:52,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:52,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:55:56,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:55:58,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:55:58,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:55:58,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:55:58,584 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:55:59,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:55:59,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:56:02,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:02,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:02,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:02,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:03,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:56:03,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:56:03,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:56:03,317 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:56:06,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:56:06,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:56:07,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:09,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:56:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:17,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:17,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:17,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:18,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:23,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:28,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:32,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:32,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:32,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:33,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:38,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:39,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:56:44,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:48,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:56:48,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:56:48,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:56:50,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:55,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:56:57,376 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_f24a9e2785b102661151.png with policy now +2023-07-12 06:56:57,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_dda90c1223ed981d3b1a.png with policy now +2023-07-12 06:56:57,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_c45fc62adfcc71af96fa.png with policy now +2023-07-12 06:56:57,406 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:56:57,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1650_3e49f286e7ccf53fddc7.png with policy now +2023-07-12 06:56:57,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:56:57,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:56:57,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:56:57,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:56:57,692 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qig2iph7-media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 06:56:57,695 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xkpfng0b-media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 06:56:57,724 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tvp3r7rj-media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 06:56:57,745 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tyrdholp-media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 06:57:00,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:02,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:02,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:02,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 06:57:02,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 06:57:02,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 06:57:02,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 06:57:02,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:57:02,978 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2vdvl5z-media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 06:57:02,988 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6who694h-media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 06:57:02,992 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/at3uw3m5-media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 06:57:02,997 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0gjv1zsf-media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 06:57:03,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:03,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:03,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:05,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:07,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:09,435 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:57:09,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:11,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:16,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:16,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:16,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:16,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:16,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:16,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:18,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:18,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:18,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:22,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:23,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:23,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:26,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:26,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:26,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:26,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:28,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:30,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:30,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:33,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:33,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:33,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:33,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:36,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:36,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:36,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:36,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:37,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:37,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:38,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:39,437 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:57:43,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:44,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:46,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:46,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:46,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:46,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:48,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:49,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:57:49,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:57:49,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:57:51,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:51,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:54,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:57:57,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:57:57,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:57:57,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:57:57,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:57:58,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:57:58,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:57:59,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:05,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:05,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:05,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:05,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:05,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:08,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:08,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:08,101 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:08,103 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:09,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:58:10,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:12,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:58:12,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:16,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:17,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:17,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:17,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:17,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:19,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:58:19,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:20,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:20,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:20,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:21,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:26,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:26,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:27,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:27,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:27,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:27,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:31,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:58:31,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:31,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:58:31,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:58:31,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:58:33,940 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:58:33,940 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:58:35,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:35,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:35,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:37,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:39,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:58:42,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:47,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:50,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:58:50,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:58:50,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:58:53,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:58:58,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:03,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:05,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:59:05,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:05,359 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:10,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:59:10,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:15,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:20,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:59:20,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:20,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:21,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:26,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_000cc75e10c8a7929ff7.png with policy now +2023-07-12 06:59:26,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,134 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_bcf133d130695d2171a4.png with policy now +2023-07-12 06:59:26,144 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,144 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_9e45cdde0ed95786481b.png with policy now +2023-07-12 06:59:26,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 06:59:26,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1660_d35268c957b3894ad085.png with policy now +2023-07-12 06:59:26,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:26,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:59:26,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:59:26,199 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:59:26,455 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h6r9t26o-media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 06:59:26,474 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uqifco14-media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 06:59:26,487 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gx027nuu-media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 06:59:26,671 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6j2t56r3-media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 06:59:27,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:30,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:59:30,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 06:59:30,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 06:59:30,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 06:59:30,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 06:59:30,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 06:59:30,593 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdfwa2wu-media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 06:59:30,593 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5d72aimf-media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 06:59:30,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xv41s7mo-media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 06:59:30,600 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xg6w7yke-media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 06:59:32,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:36,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 06:59:36,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:36,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:36,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:37,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:37,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:39,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 06:59:42,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:44,210 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:45,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:45,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:59:45,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:59:45,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:59:48,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:51,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 06:59:51,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 06:59:51,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:59:51,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:53,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 06:59:55,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 06:59:55,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 06:59:55,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 06:59:55,475 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 06:59:58,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 06:59:58,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 06:59:58,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:04,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:05,252 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:05,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:05,853 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:05,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:05,856 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:06,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:06,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:06,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:09,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:00:09,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:12,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:12,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:15,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:16,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:16,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:16,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:16,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:19,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:19,278 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:20,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:21,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:21,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:21,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:25,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:26,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:26,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:26,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:26,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:26,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:31,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:33,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:33,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:36,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:36,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:36,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:36,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:37,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:37,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:37,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:37,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:39,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:00:40,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:40,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:41,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:47,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:47,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:47,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:47,343 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:47,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:00:47,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:00:47,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:52,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:00:52,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:00:52,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:00:52,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:54,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:00:57,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:00:57,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:00:57,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:00:57,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:00:57,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:01:01,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:01:01,493 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:01:02,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:01:02,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:01:02,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:01:02,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:01:02,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:07,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:07,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:07,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:08,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:08,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:01:08,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:01:09,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:01:13,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:18,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:22,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:22,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:22,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:24,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:29,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:34,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:37,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:37,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:37,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:40,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:01:40,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:45,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:50,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:52,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:01:52,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:01:52,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:01:56,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:01:56,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,962 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_c0361568b66056400dc2.png with policy now +2023-07-12 07:01:56,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,971 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_f129804ddaa9a57cd45c.png with policy now +2023-07-12 07:01:56,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,982 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_4f3ac5bc86e6378f1a12.png with policy now +2023-07-12 07:01:56,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:01:56,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1670_f237ce678dbe9ec09de4.png with policy now +2023-07-12 07:01:57,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:01:57,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:01:57,034 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:01:57,036 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:01:57,346 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hupxmxau-media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 07:01:57,373 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/puhwr41s-media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 07:01:57,397 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v45tg3w1-media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 07:01:57,483 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yu8u3tvq-media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 07:01:57,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:01:57,737 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 07:01:57,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 07:01:57,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 07:01:57,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 07:01:57,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:01:58,154 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6oobafvm-media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 07:01:58,156 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5xqrq9q4-media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 07:01:58,188 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yrncacfi-media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 07:01:58,229 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0p7hjzcc-media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 07:02:01,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:04,750 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:06,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:06,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:07,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:07,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:07,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:09,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:02:11,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:12,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:15,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:15,868 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:15,868 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:15,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:17,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:18,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:18,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:22,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:22,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:22,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:23,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:25,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:25,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:25,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:25,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:25,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:28,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:32,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:32,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:33,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:36,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:36,263 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:36,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:36,265 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:38,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:38,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:38,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:39,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:02:39,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:39,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:39,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:46,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:47,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:47,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:47,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:47,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:02:50,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:53,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:02:53,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:02:54,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:02:54,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:02:54,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:02:55,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:02:57,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:02:57,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:02:57,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:02:57,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:00,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:00,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:01,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:06,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:07,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:07,456 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:07,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:07,458 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:08,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:08,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:09,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:03:09,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:03:09,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:09,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:12,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:15,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:17,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:17,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:17,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:17,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:17,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:22,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:22,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:23,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:24,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:24,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:03:24,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:28,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:28,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:28,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:28,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:28,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:29,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:33,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:03:33,275 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:03:33,275 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:03:33,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:03:33,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:36,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:03:36,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:03:38,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:39,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:03:39,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:39,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:44,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:49,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:54,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:03:54,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:03:54,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:03:59,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:04,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:09,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:04:09,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:09,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:10,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:15,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:20,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:24,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:04:24,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:24,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:25,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:27,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,688 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_731d1b60253f6f4063e4.png with policy now +2023-07-12 07:04:27,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,697 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_f040c34685a98d0f3850.png with policy now +2023-07-12 07:04:27,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_8d56acd02a2f045be997.png with policy now +2023-07-12 07:04:27,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:04:27,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1680_80603d23a803515f19ef.png with policy now +2023-07-12 07:04:27,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:27,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:04:27,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:04:27,763 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:04:28,047 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lvi4c5t4-media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 07:04:28,051 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x7chm46v-media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 07:04:28,064 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8fncz23m-media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 07:04:28,072 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cjqx768t-media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 07:04:31,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:32,303 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:04:32,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:32,304 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 07:04:32,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 07:04:32,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 07:04:32,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 07:04:32,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:04:32,641 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jgu6c8s8-media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 07:04:32,646 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6m93moep-media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 07:04:32,670 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k5b8samo-media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 07:04:32,738 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mefxfa6f-media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 07:04:36,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:37,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:39,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:39,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:04:39,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:39,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:04:39,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:42,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:46,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:46,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:46,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:04:46,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:04:46,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:04:47,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:52,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:04:53,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:04:53,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:04:54,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:04:54,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:04:54,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:04:57,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:04:57,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:04:57,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:04:57,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:04:58,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:00,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:00,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:03,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:07,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:07,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:07,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:07,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:07,471 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:08,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:09,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:05:09,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:09,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:09,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:13,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:14,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:14,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:17,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:17,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:17,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:17,616 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:18,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:21,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:21,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:23,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:24,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:24,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:24,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:28,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:28,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:28,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:28,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:28,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:28,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:29,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:34,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:35,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:37,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:37,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:37,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:37,735 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:39,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:05:39,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:39,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:39,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:39,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:42,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:42,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:45,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:47,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:47,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:47,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:47,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:05:49,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:05:49,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:50,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:55,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:05:55,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:05:55,195 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:05:55,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:05:56,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:05:58,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:05:58,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:05:58,555 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:05:58,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:06:00,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:03,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:06:03,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:06:03,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:06:03,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:06:03,628 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:06:03,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:06:05,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:09,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:06:10,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:10,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:10,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:10,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:06:11,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:16,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:21,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:25,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:25,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:25,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:26,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:31,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:36,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:39,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:06:40,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:40,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:40,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:41,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:46,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:51,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:55,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:06:55,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:06:55,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:06:57,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:06:57,777 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png with policy now +2023-07-12 07:06:57,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_7295168e3d00a0e3d439.png with policy now +2023-07-12 07:06:57,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_fbcc4f86c9d83ede1f5d.png with policy now +2023-07-12 07:06:57,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:06:57,823 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1690_c1341754ea0507193be5.png with policy now +2023-07-12 07:06:57,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:06:57,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:06:57,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:06:57,850 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:06:58,122 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmausocq-media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 07:06:58,174 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q9k3rukw-media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 07:06:58,189 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z3poulg0-media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 07:06:58,190 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lptwt46n-media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 07:06:59,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:06:59,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 07:06:59,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 07:06:59,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 07:06:59,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 07:06:59,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:07:00,242 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v6116vk0-media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 07:07:00,253 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u1o3mubo-media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 07:07:00,269 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8dv0not5-media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 07:07:00,290 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b9a2wp2b-media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 07:07:03,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:06,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:08,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:08,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:09,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:07:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:07:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:10,866 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:13,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:13,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:18,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:18,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:18,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:18,561 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:18,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:20,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:20,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:24,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:25,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:25,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:27,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:28,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:28,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:28,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:28,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:29,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:34,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:34,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:35,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:38,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:38,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:38,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:38,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:39,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:07:40,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:40,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:40,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:41,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:41,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:45,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:48,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:48,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:48,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:48,517 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:07:49,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:07:49,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:50,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:55,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:07:55,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:07:55,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:07:56,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:07:57,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:07:58,001 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:07:58,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:07:58,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:01,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:03,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:03,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:07,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:08,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:08,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:08,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:08,270 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:09,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:08:10,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:10,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:10,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:10,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:08:12,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:17,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:17,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:19,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:19,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:19,143 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:19,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:22,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:24,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:24,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:26,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:08:26,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:26,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:08:28,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:29,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:29,907 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:29,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:29,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:31,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:31,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:33,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:34,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:08:34,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:08:34,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:08:34,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:08:38,256 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:08:38,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:08:39,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:39,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:08:41,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:08:41,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:41,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:08:44,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:49,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:54,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:08:56,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:08:56,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:08:56,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:00,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:05,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:09,466 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:09:11,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:11,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:11,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:11,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:16,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:21,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:26,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:26,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:26,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:26,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:28,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_f5c67f9540d8b5ed5c7c.png with policy now +2023-07-12 07:09:28,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,557 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_f31d3c5e28119395907e.png with policy now +2023-07-12 07:09:28,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,570 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_508c64e8a2ff4f92e95d.png with policy now +2023-07-12 07:09:28,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:09:28,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1700_46dc3e4359141341c6d4.png with policy now +2023-07-12 07:09:28,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:28,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:09:28,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:09:28,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:09:28,871 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kmyfeqhq-media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 07:09:28,883 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2wv7yiu4-media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 07:09:28,899 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fuw4ez18-media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 07:09:28,939 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c2ooy5h6-media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 07:09:32,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:34,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:09:34,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:34,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 07:09:34,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 07:09:34,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 07:09:34,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 07:09:34,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:09:34,854 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4bvzoqzb-media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 07:09:34,857 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0r170v3m-media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 07:09:34,859 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bew0rfft-media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 07:09:34,901 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/36byuuwt-media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 07:09:38,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:38,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:39,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:09:41,525 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:41,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:41,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:41,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:43,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:48,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:48,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:48,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:09:48,256 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:09:48,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:09:48,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:09:48,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:54,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:09:55,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:09:56,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:09:56,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:09:56,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:09:57,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:09:57,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:09:57,948 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:09:57,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:09:59,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:02,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:02,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:05,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:07,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:07,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:07,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:07,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:09,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:10:09,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:09,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:10,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:12,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:12,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:12,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:15,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:16,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:17,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:17,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:17,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:17,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:21,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:23,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:23,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:26,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:27,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:27,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:27,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:28,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:28,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:28,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:28,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:30,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:30,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:32,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:37,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:37,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:38,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:38,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:38,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:38,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:39,471 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:10:43,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:43,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:43,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:43,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:44,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:44,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:48,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:48,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:48,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:48,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:48,619 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:51,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:51,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:10:53,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:58,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:10:58,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:10:58,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:10:58,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:10:58,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:10:58,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:10:58,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:10:58,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:10:58,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:10:58,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:11:03,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:11:03,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:11:03,310 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:11:03,312 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:11:03,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:05,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:11:05,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:11:08,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:09,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:11:13,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:13,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:13,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:11:14,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:19,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:24,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:28,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:28,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:28,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:11:29,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:34,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:39,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:39,473 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:11:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:43,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:11:45,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:50,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:55,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:11:57,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,479 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_bbaae7ee7d09d30b3e1a.png with policy now +2023-07-12 07:11:57,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,489 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_92fe9627890fe0330f34.png with policy now +2023-07-12 07:11:57,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_be9cd4b95a81eb492abf.png with policy now +2023-07-12 07:11:57,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:11:57,512 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1710_603a92e14eb64a33c047.png with policy now +2023-07-12 07:11:57,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:11:57,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:11:57,560 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:11:57,563 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:11:57,800 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/trl7vgin-media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 07:11:57,807 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pojo9ykr-media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 07:11:57,812 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6c8obe6u-media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 07:11:57,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/osh8gvul-media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 07:11:58,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:11:58,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:11:58,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:00,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:02,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:02,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 07:12:02,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 07:12:02,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 07:12:02,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 07:12:02,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:12:02,430 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yisusads-media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 07:12:02,437 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/anevop3q-media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 07:12:02,487 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7ddd126h-media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 07:12:02,496 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/41rkr40h-media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 07:12:05,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:07,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:09,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:09,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:12:11,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:13,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:13,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:16,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:16,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:17,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:17,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:17,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:17,521 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:21,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:23,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:23,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:26,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:27,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:27,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:27,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:27,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:28,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:28,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:30,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:30,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:32,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:37,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:37,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:37,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:37,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:37,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:37,435 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:39,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:12:42,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:43,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:44,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:44,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:47,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:47,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:47,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:47,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:47,810 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:51,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:12:51,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:53,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:12:58,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:12:58,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:12:58,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:12:58,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:12:58,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:12:58,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:12:58,300 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:12:58,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:12:59,303 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:05,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:05,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:05,355 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:08,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:08,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:08,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:08,805 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:09,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:13:10,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:12,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:12,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:13,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:13,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:13:13,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:15,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:18,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:18,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:18,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:18,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:19,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:19,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:21,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:26,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:26,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:28,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:28,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:28,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:28,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:28,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:28,499 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:32,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:33,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:13:33,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:13:33,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:13:33,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:13:33,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:13:33,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:37,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:39,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:13:40,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:13:43,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:13:43,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:43,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:43,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:48,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:53,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:13:58,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:13:58,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:13:58,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:13:58,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:03,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:08,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:09,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:14:13,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:13,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:13,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:14:14,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:19,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:24,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:27,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,820 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_80be20563f1620981e46.png with policy now +2023-07-12 07:14:27,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,830 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_2038713cb0876e29c6c8.png with policy now +2023-07-12 07:14:27,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_95d7597f938efa3e944e.png with policy now +2023-07-12 07:14:27,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:14:27,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1720_551525fb129586cd0c60.png with policy now +2023-07-12 07:14:27,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:27,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:14:27,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:14:27,895 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:14:28,116 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9rljlkhl-media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 07:14:28,141 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vd7hdrm4-media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 07:14:28,141 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ai28rglx-media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 07:14:28,171 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ew6jskpo-media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 07:14:28,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:28,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:28,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:14:29,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:14:29,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 07:14:29,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 07:14:29,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 07:14:29,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 07:14:29,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:14:30,018 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6mr8n795-media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 07:14:30,022 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xnpe6f38-media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 07:14:30,040 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wtg8jpjy-media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 07:14:30,110 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k0x370pl-media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 07:14:30,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:35,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:36,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:37,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:39,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:14:40,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:43,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:43,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:43,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:43,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:14:46,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:46,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:46,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:14:46,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:14:46,786 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:14:50,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:14:50,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:52,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:56,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:14:56,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:14:56,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:14:56,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:14:57,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:14:57,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:14:57,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:14:58,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:14:58,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:14:58,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:03,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:04,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:06,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:06,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:06,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:06,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:08,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:09,482 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:15:11,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:11,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:14,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:14,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:14,084 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:14,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:17,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:17,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:17,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:17,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:18,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:18,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:19,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:24,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:25,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:27,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:27,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:27,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:27,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:29,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:29,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:29,380 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:29,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:32,924 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:32,925 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:35,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:37,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:37,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:37,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:37,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:39,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:15:39,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:39,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:40,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:44,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:44,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:44,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:15:46,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:46,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:46,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:46,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:46,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:46,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:15:46,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:51,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:53,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:15:56,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:15:57,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:15:57,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:15:57,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:15:57,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:15:59,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:15:59,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:15:59,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:00,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:16:00,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:16:02,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:02,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:16:02,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:16:02,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:16:02,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:16:07,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:07,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:16:07,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:16:09,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:16:13,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:14,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:14,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:14,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:18,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:23,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:28,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:29,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:29,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:29,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:34,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:39,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:16:40,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:44,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:44,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:44,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:16:45,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:50,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:55,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:16:56,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_700fef9298b995bd049b.png with policy now +2023-07-12 07:16:56,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_9fb3356673dd42522db4.png with policy now +2023-07-12 07:16:56,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,651 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_a001a16142d0cdf3dd8e.png with policy now +2023-07-12 07:16:56,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:16:56,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1730_62bcdb68c12bb4eee832.png with policy now +2023-07-12 07:16:56,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:16:56,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:16:56,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:16:56,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:16:56,964 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zb40lzma-media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 07:16:56,972 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9rh15ulw-media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 07:16:56,978 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i1hxdi0u-media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 07:16:56,995 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2edkeqe2-media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 07:16:57,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:16:57,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 07:16:57,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 07:16:57,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 07:16:57,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 07:16:57,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:16:57,573 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/flm9oqff-media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 07:16:57,585 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4habb0yv-media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 07:16:57,586 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hs2yc8ck-media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 07:16:57,593 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m2ciyw1m-media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 07:16:59,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:16:59,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:16:59,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:00,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:04,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:06,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:06,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:09,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:17:11,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:11,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:14,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:17:14,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:17:14,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:16,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:16,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:16,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:16,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:16,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:18,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:18,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:22,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:25,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:25,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:25,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:25,960 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:25,961 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:27,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:30,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:17:30,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:17:30,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:32,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:32,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:33,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:35,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:35,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:35,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:35,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:38,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:39,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:39,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:39,487 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:17:44,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:17:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:17:45,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:17:46,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:46,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:46,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:46,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:17:46,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:17:46,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:50,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:53,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:17:55,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:17:56,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:17:56,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:17:56,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:17:56,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:00,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:00,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:00,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:00,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:00,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:00,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:06,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:06,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:06,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:06,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:06,191 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:09,488 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:18:12,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:14,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:15,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:15,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:15,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:16,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:16,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:16,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:16,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:17,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:21,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:21,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:22,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:26,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:26,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:26,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:26,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:28,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:28,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:30,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:30,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:30,875 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:31,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:18:31,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:18:31,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:18:31,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:18:33,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:35,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:18:35,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:18:38,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:39,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:18:43,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:45,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:18:45,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:18:45,875 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:18:48,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:53,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:18:58,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:00,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:00,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:00,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:04,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:09,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:09,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:19:14,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:15,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:15,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:15,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:20,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:25,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:25,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_4dc54de8f062b7449876.png with policy now +2023-07-12 07:19:25,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_9ead6b955150561d6d4c.png with policy now +2023-07-12 07:19:25,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_d91846de1cf74e316293.png with policy now +2023-07-12 07:19:25,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:19:25,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1740_bdda721ad2fd6569a964.png with policy now +2023-07-12 07:19:25,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:25,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:19:25,993 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:19:25,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:19:26,276 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/me3o9r19-media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 07:19:26,286 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sjybkwig-media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 07:19:26,293 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ri3lz2z-media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 07:19:26,323 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/arh639nj-media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 07:19:30,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:31,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:31,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:31,165 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:31,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:19:31,823 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 07:19:31,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 07:19:31,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 07:19:31,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 07:19:31,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:19:32,282 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/812paalf-media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 07:19:32,288 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jsd7m6an-media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 07:19:32,308 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yh8v3jgi-media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 07:19:32,324 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k67yy6wd-media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 07:19:35,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:36,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:38,837 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:39,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:19:42,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:44,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:44,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:19:44,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:19:44,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:19:46,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:19:46,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:19:46,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:19:46,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:19:46,852 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:48,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:53,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:19:54,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:19:54,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:19:54,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:19:54,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:19:54,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:19:54,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:19:59,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:02,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:02,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:02,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:02,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:04,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:05,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:05,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:05,484 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:05,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:09,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:20:10,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:10,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:10,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:15,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:16,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:16,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:16,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:16,444 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:17,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:17,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:17,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:18,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:18,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:21,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:26,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:26,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:26,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:26,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:26,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:26,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:26,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:32,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:32,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:32,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:32,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:35,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:36,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:36,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:36,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:36,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:37,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:39,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:20:43,097 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:43,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:43,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:47,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:20:47,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:20:47,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:20:47,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:47,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:47,328 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:47,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:49,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:51,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:51,112 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:54,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:20:57,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:20:57,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:20:57,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:20:57,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:20:59,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:20:59,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:20:59,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:02,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:21:02,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:21:02,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:21:02,014 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:21:02,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:02,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:02,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:04,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:07,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:21:07,140 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:21:09,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:21:10,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:15,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:17,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:17,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:17,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:21,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:26,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:31,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:32,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:32,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:32,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:36,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:39,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:21:41,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:46,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:47,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:21:47,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:21:47,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:21:52,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:21:56,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_1d177e30435b1c654b3d.png with policy now +2023-07-12 07:21:56,558 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,559 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png with policy now +2023-07-12 07:21:56,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_06dd5b28c3be7a9bbdb9.png with policy now +2023-07-12 07:21:56,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:21:56,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1750_94c6bfdf23b465d2a61f.png with policy now +2023-07-12 07:21:56,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:21:56,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:21:56,628 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:21:56,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:21:56,868 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/enpsc35o-media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 07:21:56,876 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e14us6j3-media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 07:21:56,880 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/egy1o4xm-media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 07:21:56,893 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xotiz8n9-media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 07:21:57,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:02,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:02,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:02,366 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:02,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:03,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:03,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:03,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 07:22:03,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 07:22:03,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 07:22:03,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 07:22:03,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:22:03,719 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wk6urojj-media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 07:22:03,725 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/23r2vrnb-media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 07:22:03,741 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x9ih07jo-media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 07:22:03,747 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wp4xrraw-media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 07:22:07,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:08,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:09,498 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:22:11,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:13,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:15,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:15,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:15,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:15,905 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:18,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:18,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:18,195 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:18,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:19,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:19,405 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:24,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:25,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:25,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:25,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:25,913 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:27,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:27,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:30,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:33,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:33,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:33,267 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:35,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:35,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:36,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:36,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:36,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:36,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:39,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:22:41,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:43,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:43,445 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:46,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:46,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:46,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:46,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:46,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:48,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:22:48,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:22:48,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:22:51,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:51,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:22:51,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:56,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:22:56,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:22:56,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:22:56,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:22:56,642 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:22:59,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:22:59,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:01,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:03,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:03,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:03,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:07,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:07,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:07,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:07,141 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:07,143 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:07,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:07,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:09,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:23:12,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:15,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:17,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:17,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:17,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:17,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:18,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:19,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:19,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:19,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:23,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:23,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:23,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:27,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:27,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:27,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:27,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:28,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:31,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:31,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:32,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:23:32,849 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:23:32,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:23:32,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:23:34,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:34,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:34,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:34,662 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:39,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:23:39,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:39,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:23:39,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:23:44,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:49,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:49,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:23:49,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:23:49,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:23:54,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:23:59,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:04,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:04,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:04,763 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:05,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:09,503 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:24:11,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:16,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:19,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:19,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:19,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:21,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:26,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:27,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_88b7298825f502e5a062.png with policy now +2023-07-12 07:24:27,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_0d6a954f28a281ccb3e6.png with policy now +2023-07-12 07:24:27,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,109 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png with policy now +2023-07-12 07:24:27,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:24:27,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1760_e4640ecdc8c22558e54b.png with policy now +2023-07-12 07:24:27,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:27,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:24:27,160 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:24:27,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:24:27,477 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpoyoq11-media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 07:24:27,480 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nvi9jur1-media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 07:24:27,482 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b6iclmr6-media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 07:24:27,521 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rnwpoach-media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 07:24:27,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:24:27,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 07:24:27,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 07:24:27,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 07:24:27,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 07:24:27,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:24:28,358 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tc6ejixr-media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 07:24:28,364 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/crbroj10-media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 07:24:28,365 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sva7a314-media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 07:24:28,369 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/urrbtt48-media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 07:24:32,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:34,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:34,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:34,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:35,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:24:37,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:37,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:39,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:24:43,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:43,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:24:46,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:46,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:24:46,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:24:46,958 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:24:48,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:49,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:24:49,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:24:49,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:24:51,979 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:24:51,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:24:54,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:24:56,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:24:56,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:24:56,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:24:56,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:24:59,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:24:59,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:00,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:05,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:25:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:05,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:06,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:07,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:07,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:07,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:07,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:08,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:08,009 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:09,505 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:25:11,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:16,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:16,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:17,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:17,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:17,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:17,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:19,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:20,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:22,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:24,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:24,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:27,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:27,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:27,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:27,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:27,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:32,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:32,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:33,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:34,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:35,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:37,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:37,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:37,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:37,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:38,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:39,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:25:40,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:40,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:44,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:48,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:48,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:48,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:48,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:48,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:25:50,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:25:50,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:25:50,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:25:50,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:55,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:25:56,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:25:56,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:25:57,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:25:57,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:25:57,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:25:57,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:26:01,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:02,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:26:02,679 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:26:02,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:26:02,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:26:04,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:26:04,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:26:05,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:05,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:05,348 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:07,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:09,508 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:26:12,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:17,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:20,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:20,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:20,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:23,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:28,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:33,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:35,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:35,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:35,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:38,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:39,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:26:44,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:49,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:50,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:26:50,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:26:50,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:26:54,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:26:56,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_c9724817961bea50e2eb.png with policy now +2023-07-12 07:26:56,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,881 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_f3ed24daee483f81fe21.png with policy now +2023-07-12 07:26:56,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,891 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_6f1689c0e9f7dc8526d4.png with policy now +2023-07-12 07:26:56,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:26:56,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1770_f03b061475b754e24cb5.png with policy now +2023-07-12 07:26:56,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:26:56,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:26:56,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:26:56,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:26:57,166 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m53zuuw2-media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 07:26:57,185 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/95blgi92-media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 07:26:57,227 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iwqbdmxa-media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 07:26:57,246 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/co7jg7gh-media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 07:26:59,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:00,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:00,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 07:27:00,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 07:27:00,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 07:27:00,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 07:27:00,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:27:00,817 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/31lp74ho-media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 07:27:00,852 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n7ik9cq0-media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 07:27:00,854 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2cz9xul7-media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 07:27:00,869 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yy9yqxyh-media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 07:27:04,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:05,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:27:05,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:05,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:06,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:08,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:09,510 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:27:10,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:15,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:15,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:15,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:15,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:16,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:16,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:16,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:20,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:27:20,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:20,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:21,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:24,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:25,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:25,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:25,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:25,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:27,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:32,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:32,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:32,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:35,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:35,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:35,490 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:35,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:35,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:35,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:37,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:39,511 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:27:40,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:40,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:42,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:45,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:45,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:45,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:45,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:48,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:48,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:48,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:50,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:27:50,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:27:53,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:27:56,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:27:56,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:27:56,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:27:56,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:27:56,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:27:56,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:27:58,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:04,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:04,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:06,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:06,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:06,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:06,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:06,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:06,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:06,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:09,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:28:09,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:12,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:12,786 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:15,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:17,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:17,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:17,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:17,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:20,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:20,798 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:20,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:21,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:21,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:21,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:26,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:28,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:28,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:28,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:28,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:28,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:28,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:31,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:32,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:28:32,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:28:32,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:28:33,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:28:36,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:28:36,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:28:36,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:36,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:36,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:37,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:39,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:28:42,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:47,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:51,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:28:51,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:28:51,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:28:53,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:28:58,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:03,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:07,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:07,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:07,157 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:08,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:09,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:29:13,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:18,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:22,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:22,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:22,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:24,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:27,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_e0be2e3dcde4725c11f7.png with policy now +2023-07-12 07:29:27,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_43277f4f939e843ab8c9.png with policy now +2023-07-12 07:29:27,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_437f85ccbcb8672d8508.png with policy now +2023-07-12 07:29:27,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:29:27,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1780_19d50aae0838de593b50.png with policy now +2023-07-12 07:29:27,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:27,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:29:27,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:29:27,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:29:27,552 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wgmt0qfb-media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 07:29:27,577 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yfz1hu8t-media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 07:29:27,595 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fzdgkf49-media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 07:29:27,626 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0u9b2c6t-media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 07:29:29,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:33,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:29:33,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:33,070 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 07:29:33,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 07:29:33,071 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 07:29:33,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 07:29:33,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:29:33,422 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fqm1r4d6-media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 07:29:33,435 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p3v4zms5-media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 07:29:33,474 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jkz1cilb-media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 07:29:34,314 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6tdvlxeq-media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 07:29:35,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:36,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:37,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:37,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:37,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:39,516 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:29:41,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:41,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:45,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:45,785 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:29:45,785 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:29:45,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:29:46,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:49,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:29:49,099 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:52,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:29:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:29:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:29:52,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:29:55,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:29:55,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:29:55,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:29:55,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:29:57,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:29:57,113 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:29:57,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:03,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:05,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:05,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:05,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:05,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:05,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:08,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:08,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:08,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:09,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:30:09,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:13,141 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:13,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:14,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:16,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:16,682 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:16,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:16,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:20,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:23,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:23,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:23,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:25,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:26,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:26,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:26,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:26,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:29,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:29,166 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:31,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:36,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:37,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:37,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:37,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:37,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:37,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:37,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:39,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:39,518 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:30:41,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:45,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:47,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:47,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:47,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:47,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:47,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:30:52,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:53,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:30:53,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:30:54,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:30:54,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:30:54,529 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:30:57,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:30:57,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:30:57,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:30:57,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:30:57,746 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:31:01,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:31:01,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:31:02,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:31:02,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:31:02,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:31:02,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:31:03,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:08,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:09,400 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:31:09,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:31:09,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:31:09,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:09,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:09,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:13,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:18,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:23,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:24,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:24,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:24,674 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:28,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:33,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:38,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:39,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:31:39,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:39,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:39,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:44,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:49,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:54,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:31:54,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:31:54,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:31:54,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:31:56,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:56,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_1fbb3840c9621213d4a8.png with policy now +2023-07-12 07:31:56,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:56,994 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_6c8d3f0f2e6689c4c999.png with policy now +2023-07-12 07:31:57,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:57,004 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_73398702cb61fca4ba23.png with policy now +2023-07-12 07:31:57,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:31:57,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1790_86559286380435a179bf.png with policy now +2023-07-12 07:31:57,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:31:57,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:31:57,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:31:57,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:31:57,299 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bnl9hdpd-media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 07:31:57,310 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vt37c6a2-media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 07:31:57,329 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e5wu5a9s-media/images/validation_1790_86559286380435a179bf.png +2023-07-12 07:31:57,353 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1ptuv03-media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 07:31:57,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:31:57,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 07:31:57,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 07:31:57,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_86559286380435a179bf.png +2023-07-12 07:31:57,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 07:31:57,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:31:58,058 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s0419ven-media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 07:31:58,088 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vy9v16li-media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 07:31:58,098 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8wmvpyud-media/images/validation_1790_86559286380435a179bf.png +2023-07-12 07:31:58,105 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d4lxbhr0-media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 07:32:00,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:05,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:05,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:07,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:09,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:32:09,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:09,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:09,882 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:10,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:13,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:16,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:16,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:16,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:16,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:16,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:21,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:21,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:22,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:25,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:25,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:25,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:26,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:26,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:26,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:26,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:28,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:29,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:29,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:33,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:36,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:36,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:36,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:36,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:37,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:37,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:38,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:39,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:32:40,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:40,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:40,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:44,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:45,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:46,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:46,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:46,698 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:46,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:49,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:53,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:32:53,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:32:54,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:32:56,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:32:56,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:32:56,603 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:32:57,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:32:57,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:32:57,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:32:57,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:32:59,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:01,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:01,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:05,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:07,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:07,980 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:07,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:07,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:09,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:33:09,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:09,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:10,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:12,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:12,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:12,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:15,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:17,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:18,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:18,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:18,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:18,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:20,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:25,956 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:25,957 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:26,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:27,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:27,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:27,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:28,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:28,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:28,041 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:28,043 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:32,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:32,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:33:32,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:33:32,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:33:32,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:33:33,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:33:33,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:38,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:39,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:33:41,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:33:42,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:42,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:42,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:44,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:49,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:54,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:33:57,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:33:57,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:33:57,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:33:59,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:04,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:09,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:34:10,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:12,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:12,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:12,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:16,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:21,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:26,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:27,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_1b5a6f042a0a4ba4a34a.png with policy now +2023-07-12 07:34:27,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_3003f5b283d5c2f3d3a3.png with policy now +2023-07-12 07:34:27,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_a5df1ab61da66e19d024.png with policy now +2023-07-12 07:34:27,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:34:27,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1800_c6b5b46f40b7e3cc06e5.png with policy now +2023-07-12 07:34:27,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:27,442 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:34:27,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:34:27,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:34:27,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:27,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:27,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:27,712 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lscsiwz4-media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 07:34:27,714 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wch5qjb7-media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 07:34:27,724 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q66ef5yj-media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 07:34:27,765 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mkrzwwhu-media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 07:34:30,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 07:34:30,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 07:34:30,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:34:30,556 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/til5tiea-media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 07:34:30,563 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qy6vriyq-media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 07:34:30,564 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e9vbjqij-media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 07:34:30,580 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9fm9xo1m-media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 07:34:32,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:37,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:37,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:38,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:34:39,527 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:34:42,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:42,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:42,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:42,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:46,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:34:47,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:47,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:34:47,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:34:47,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:34:48,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:53,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:34:54,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:34:54,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:34:57,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:34:57,448 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:34:57,448 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:34:57,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:34:57,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:34:57,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:34:57,543 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:34:58,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:02,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:02,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:04,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:07,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:07,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:07,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:07,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:09,528 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:35:09,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:10,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:10,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:12,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:12,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:12,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:14,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:18,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:18,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:18,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:18,051 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:18,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:18,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:20,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:25,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:26,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:27,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:27,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:27,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:29,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:29,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:29,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:29,772 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:30,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:34,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:34,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:35,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:39,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:35:40,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:40,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:40,064 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:40,066 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:41,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:42,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:42,513 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:42,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:42,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:42,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:46,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:49,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:35:49,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:35:49,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:35:49,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:35:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:35:50,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:35:52,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:57,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:35:57,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:35:57,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:35:57,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:35:58,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:36:00,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:36:00,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:36:00,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:36:00,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:36:03,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:04,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:36:04,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:36:04,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:36:04,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:36:06,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:36:06,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:36:09,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:09,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:36:12,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:12,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:12,559 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:14,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:36:14,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:19,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:24,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:27,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:27,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:27,559 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:30,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:35,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:39,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:36:41,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:42,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:42,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:42,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:46,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:51,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:36:57,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:36:57,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:36:57,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:36:59,296 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_1d8ecdbc961ca85f19f9.png with policy now +2023-07-12 07:36:59,307 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,307 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_22b0716ab99e4f2c1baf.png with policy now +2023-07-12 07:36:59,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,318 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_db05126b5d482e1f86c1.png with policy now +2023-07-12 07:36:59,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:36:59,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1810_e9390a064f93495e881d.png with policy now +2023-07-12 07:36:59,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:36:59,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:36:59,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:36:59,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:36:59,650 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kpp2r0tp-media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 07:36:59,654 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t56qqajn-media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 07:36:59,702 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b6ex3ggg-media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 07:36:59,724 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8d4y3jcx-media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 07:37:02,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:02,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:02,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:02,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 07:37:02,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 07:37:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 07:37:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 07:37:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:37:03,187 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1qi9us5i-media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 07:37:03,198 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fw3pwp8l-media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 07:37:03,200 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9un616ma-media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 07:37:03,201 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/07g5gvxl-media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 07:37:07,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:09,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:09,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:37:10,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:13,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:13,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:13,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:13,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:18,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:18,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:18,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:18,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:18,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:18,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:23,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:26,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:26,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:28,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:28,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:28,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:29,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:29,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:29,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:29,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:29,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:34,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:34,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:34,876 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:39,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:37:39,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:39,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:39,860 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:39,860 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:39,862 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:42,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:42,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:44,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:44,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:44,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:44,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:50,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:50,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:50,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:50,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:50,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:37:50,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:37:50,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:56,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:37:58,920 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:37:59,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:37:59,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:37:59,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:37:59,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:37:59,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:37:59,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:37:59,763 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:01,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:06,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:07,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:07,095 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:09,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:38:10,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:10,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:10,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:10,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:12,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:14,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:14,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:14,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:38:15,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:15,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:17,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:21,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:21,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:21,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:21,849 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:22,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:23,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:23,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:27,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:29,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:29,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:29,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:38:31,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:31,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:31,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:31,868 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:31,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:33,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:36,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:38:36,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:38:36,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:38:36,603 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:38:39,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:38:39,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:38:39,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:39,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:38:44,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:44,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:44,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:38:44,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:49,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:54,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:38:59,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:38:59,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:38:59,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:00,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:05,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:09,537 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:39:10,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:14,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:14,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:14,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:16,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:21,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:26,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:29,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:29,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:29,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:30,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_0da500ef601e86506941.png with policy now +2023-07-12 07:39:30,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_7ecf7da22bc3b3fad59d.png with policy now +2023-07-12 07:39:30,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_3619de4ba8713b9bd91c.png with policy now +2023-07-12 07:39:30,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:39:30,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1820_953f6fe5b812a7050a46.png with policy now +2023-07-12 07:39:30,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:30,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:39:30,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:39:30,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:39:31,093 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ykyc3xt2-media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 07:39:31,098 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/77gmv7y5-media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 07:39:31,124 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gln7kaw4-media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 07:39:31,136 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jehgk0vs-media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 07:39:31,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:35,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:39:35,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:35,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 07:39:35,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 07:39:35,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 07:39:35,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 07:39:35,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:39:35,794 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s3pb70z6-media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 07:39:35,802 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bxw75m71-media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 07:39:35,808 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v9xl1yck-media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 07:39:35,809 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eolfm3fx-media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 07:39:37,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:39,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:39:40,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:43,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:43,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:44,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:44,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:44,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:48,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:49,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:49,483 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:39:49,483 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:39:49,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:39:51,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:39:51,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:53,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:59,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:39:59,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:39:59,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:39:59,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:39:59,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:39:59,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:39:59,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:39:59,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:39:59,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:04,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:07,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:09,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:09,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:40:10,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:10,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:10,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:10,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:14,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:40:14,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:40:14,800 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:40:14,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:15,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:15,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:20,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:20,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:20,270 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:20,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:20,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:23,534 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:23,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:25,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:30,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:30,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:40:30,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:40:30,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:40:31,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:31,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:31,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:31,552 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:31,554 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:35,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:39,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:40:39,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:39,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:41,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:42,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:42,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:42,198 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:42,200 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:45,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:40:45,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:40:47,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:47,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:47,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:52,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:40:52,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:40:52,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:40:52,328 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:40:52,330 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:40:55,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:40:55,748 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:40:57,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:02,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:02,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:02,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:02,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:02,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:41:02,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:41:02,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:41:02,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:41:03,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:41:03,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:41:06,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:41:06,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:41:06,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:41:06,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:41:08,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:09,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:41:11,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:41:11,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:41:14,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:17,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:17,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:17,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:20,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:25,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:30,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:32,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:32,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:32,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:35,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:39,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:41:40,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:45,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:47,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:41:47,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:41:47,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:41:51,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:41:56,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:01,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_7b892c3791e1a5536a80.png with policy now +2023-07-12 07:42:01,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_3cff7354bef942c8de30.png with policy now +2023-07-12 07:42:01,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_7b9c6d907af1a36abe9c.png with policy now +2023-07-12 07:42:01,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:42:01,247 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1830_d2e0fe71542ddbda5921.png with policy now +2023-07-12 07:42:01,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:01,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:01,292 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:01,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:01,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:01,550 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wp67pqbx-media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 07:42:01,551 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z8wys8t8-media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 07:42:01,565 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gvm63afe-media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 07:42:01,567 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2lm31y3d-media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 07:42:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:02,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:07,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:08,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:08,026 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:08,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 07:42:08,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 07:42:08,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 07:42:08,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 07:42:08,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:42:08,443 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rysdvin3-media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 07:42:08,452 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ps52ytx-media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 07:42:08,458 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e1sougx5-media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 07:42:08,466 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bqdevlby-media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 07:42:09,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:42:11,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:12,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:16,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:17,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:17,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:17,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:18,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:20,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:20,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:20,912 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:20,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:23,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:24,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:24,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:29,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:30,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:30,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:30,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:30,419 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:32,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:32,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:32,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:32,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:32,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:34,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:39,544 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:42:39,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:40,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:41,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:41,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:41,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:41,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:45,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:47,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:42:47,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:42:47,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:42:48,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:48,098 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:51,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:42:52,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:42:52,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:42:52,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:42:52,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:42:56,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:42:56,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:42:57,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:02,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:02,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:02,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:02,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:02,216 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:02,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:43:02,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:02,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:04,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:04,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:07,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:09,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:43:12,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:12,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:12,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:12,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:12,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:12,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:17,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:43:17,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:17,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:17,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:20,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:20,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:23,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:23,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:23,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:23,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:23,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:28,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:28,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:28,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:32,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:32,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:33,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:33,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:33,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:33,505 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:33,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:36,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:36,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:38,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:43:38,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:43:38,120 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:43:38,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:43:38,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:39,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:43:44,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:43:44,353 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:43:44,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:47,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:43:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:43:47,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:43:50,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:43:55,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:00,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:02,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:02,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:02,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:06,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:09,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:44:11,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:16,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:17,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:17,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:17,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:22,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:27,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:32,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:32,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_187a1a1f0c43344a0712.png with policy now +2023-07-12 07:44:32,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,539 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_31e70925ba342cb958b8.png with policy now +2023-07-12 07:44:32,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,549 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_2f7e6e14419630be9783.png with policy now +2023-07-12 07:44:32,559 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:44:32,559 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1840_b75b67c6b0d3f3cd134b.png with policy now +2023-07-12 07:44:32,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 07:44:32,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 07:44:32,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 07:44:32,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 07:44:32,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:44:32,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:44:32,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:44:32,600 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:44:32,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:44:32,859 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ztyw3cuk-media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 07:44:32,865 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k4lh9ym0-media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 07:44:32,897 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/irxmb2xs-media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 07:44:32,898 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6h2ws7hw-media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 07:44:32,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:32,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:32,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:33,299 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2o2jyzh9-media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 07:44:33,314 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xe429q4w-media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 07:44:33,713 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s5cp6p73-media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 07:44:33,751 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/54wz4pvo-media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 07:44:37,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:39,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:44:40,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:44:40,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:44:42,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:44:43,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:44:47,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:44:47,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:44:48,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:44:49,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:52,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:44:52,373 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:44:52,373 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:44:52,375 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:44:54,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:44:56,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:44:56,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:44:59,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:01,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:01,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:01,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:01,976 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:03,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:03,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:03,658 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:04,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:04,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:05,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:09,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:45:10,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:11,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:11,637 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:11,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:11,639 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:12,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:12,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:15,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:19,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:19,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:19,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:20,673 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:20,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:22,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:22,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:22,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:22,232 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:25,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:28,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:28,689 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:31,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:31,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:31,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:31,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:31,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:34,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:34,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:34,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:36,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:36,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:36,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:39,550 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:45:41,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:41,837 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:41,837 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:41,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:42,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:44,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:44,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:48,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:49,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:45:49,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:45:49,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:45:52,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:45:52,573 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:45:52,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:45:52,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:45:52,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:45:52,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:45:53,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:45:58,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:00,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:46:03,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:46:03,422 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:46:03,422 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:46:03,424 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:46:04,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:05,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:46:05,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:05,226 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:07,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:46:07,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:46:07,957 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:46:07,958 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:46:08,922 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:46:08,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:46:09,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:46:09,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:14,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:16,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:46:19,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:20,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:20,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:46:20,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:25,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:30,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:35,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:35,226 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:35,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:39,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:46:40,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:45,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:50,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:46:50,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:46:50,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:46:51,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:46:56,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:01,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:02,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,269 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_82f6b3c68a48fade34f5.png with policy now +2023-07-12 07:47:02,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_3a3b446f556528dde605.png with policy now +2023-07-12 07:47:02,288 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_ec640dd671161721e4f7.png with policy now +2023-07-12 07:47:02,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:47:02,298 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1850_c776faecdec7dde7f595.png with policy now +2023-07-12 07:47:02,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:02,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:02,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:02,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:02,595 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0h30y3m3-media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 07:47:02,595 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cuv84jfo-media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 07:47:02,613 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ea6r8vdc-media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 07:47:02,635 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/12jqscka-media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 07:47:05,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:05,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 07:47:05,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 07:47:05,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 07:47:05,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 07:47:05,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:47:05,506 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6fu3y29p-media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 07:47:05,529 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gfuoc84r-media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 07:47:05,533 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y7va5w5v-media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 07:47:05,554 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bote0wz2-media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 07:47:05,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:05,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:05,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:06,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:09,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:47:11,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:12,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:13,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:17,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:21,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:21,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:21,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:21,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:22,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:22,321 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:22,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:22,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:23,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:28,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:29,220 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:29,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:31,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:31,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:31,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:31,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:34,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:36,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:36,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:36,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:37,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:37,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:39,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:39,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:47:41,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:41,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:41,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:41,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:45,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:45,248 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:45,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:50,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:47:51,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:47:51,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:47:51,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:47:52,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:47:52,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:47:52,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:47:52,405 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:47:53,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:47:53,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:47:56,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:01,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:01,277 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:03,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:03,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:03,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:03,270 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:06,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:06,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:06,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:06,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:09,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:09,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:09,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:48:11,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:14,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:14,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:14,085 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:14,087 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:17,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:17,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:17,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:21,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:21,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:21,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:22,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:24,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:24,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:24,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:24,196 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:25,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:25,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:28,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:33,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:33,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:34,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:34,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:34,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:34,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:36,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:36,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:36,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:39,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:48:39,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:39,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:48:39,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:48:39,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:48:39,558 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:48:41,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:48:41,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:44,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:49,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:48:49,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:51,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:48:51,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:48:51,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:48:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:48:59,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:04,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:06,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:06,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:06,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:09,560 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:49:10,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:15,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:20,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:21,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:21,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:21,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:25,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:30,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:33,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_7b660814a00507637354.png with policy now +2023-07-12 07:49:33,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_2bee3f76993cc3a6ba8f.png with policy now +2023-07-12 07:49:33,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_2ec1726523e7df6ef87a.png with policy now +2023-07-12 07:49:33,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:49:33,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1860_763a0499812940f346e6.png with policy now +2023-07-12 07:49:33,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:49:33,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:49:33,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:49:33,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:49:33,460 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/glfcdekb-media/images/validation_1860_7b660814a00507637354.png +2023-07-12 07:49:33,482 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6968sm3s-media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 07:49:33,503 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/erdw3tvn-media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 07:49:33,504 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dxb4e7uq-media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 07:49:36,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:37,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:37,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:37,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:37,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:49:37,766 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 07:49:37,767 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_7b660814a00507637354.png +2023-07-12 07:49:37,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 07:49:37,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 07:49:37,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:49:38,125 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vw28brbv-media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 07:49:38,134 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/17bhwyml-media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 07:49:38,134 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vuor62zl-media/images/validation_1860_7b660814a00507637354.png +2023-07-12 07:49:38,186 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9kxak9dk-media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 07:49:39,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:49:42,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:42,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:49:45,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:49:48,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:51,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:49:51,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:49:51,535 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:49:51,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:49:52,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:49:52,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:49:52,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:49:53,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:49:53,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:49:53,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:49:59,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:01,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:01,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:01,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:01,650 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:01,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:01,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:04,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:07,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:07,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:07,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:09,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:50:09,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:09,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:11,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:11,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:11,428 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:11,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:14,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:17,838 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:17,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:20,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:22,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:22,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:22,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:22,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:22,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:22,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:22,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:25,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:25,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:30,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:32,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:32,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:32,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:32,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:33,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:33,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:36,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:37,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:37,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:37,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:39,563 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:50:41,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:42,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:43,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:43,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:43,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:43,238 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:47,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:50,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:50,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:50:52,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:50:52,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:50:52,834 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:50:52,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:53,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:50:53,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:50:53,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:50:53,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:50:57,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:50:58,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:50:58,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:51:03,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:04,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:51:04,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:51:04,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:51:04,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:51:06,083 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:51:06,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:51:08,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:08,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:08,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:08,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:51:08,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:51:08,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:51:08,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:51:09,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:09,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:51:14,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:51:14,100 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:51:14,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:19,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:23,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:23,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:23,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:25,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:30,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:35,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:38,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:38,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:38,050 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:39,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:51:40,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:45,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:50,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:51:53,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:51:53,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:51:53,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:51:56,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:01,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:02,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,486 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_c031f9ce29973bad20f5.png with policy now +2023-07-12 07:52:02,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,529 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_386744990d75b0d3adfe.png with policy now +2023-07-12 07:52:02,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,530 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_96924703d0889517e752.png with policy now +2023-07-12 07:52:02,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:52:02,531 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1870_b0e70d9482eb144ce2cd.png with policy now +2023-07-12 07:52:02,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:02,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:02,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:02,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:02,830 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/52h52k29-media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 07:52:02,874 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ap2wy70a-media/images/validation_1870_96924703d0889517e752.png +2023-07-12 07:52:02,876 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hcn5wvtl-media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 07:52:02,884 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kdup5iap-media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 07:52:06,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:08,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:52:08,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:08,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:09,566 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:52:10,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 07:52:10,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_96924703d0889517e752.png +2023-07-12 07:52:10,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 07:52:10,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 07:52:10,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:52:10,700 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cpo472zj-media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 07:52:10,706 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dq263vvf-media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 07:52:10,731 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nyz7hxep-media/images/validation_1870_96924703d0889517e752.png +2023-07-12 07:52:10,731 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wh8s8xxp-media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 07:52:12,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:12,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:17,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:18,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:21,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:21,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:21,650 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:21,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:23,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:52:23,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:23,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:23,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:26,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:26,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:28,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:31,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:31,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:31,332 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:31,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:33,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:34,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:34,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:38,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:39,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:52:39,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:39,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:39,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:52:42,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:42,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:42,113 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:42,114 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:42,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:42,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:44,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:49,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:50,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:52:53,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:52:53,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:52:53,717 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:52:53,719 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:52:54,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:52:54,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:52:55,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:52:58,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:52:58,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:00,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:03,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:03,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:03,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:03,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:05,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:06,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:06,454 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:09,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:09,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:09,568 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:53:11,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:13,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:13,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:13,681 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:13,683 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:14,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:14,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:17,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:22,265 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:22,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:23,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:23,981 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:23,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:23,983 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:24,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:24,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:27,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:30,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:30,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:32,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:34,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:34,401 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:34,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:34,402 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:38,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:38,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:38,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:38,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:53:38,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:53:38,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:53:38,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:53:39,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:39,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:39,570 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:53:44,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:46,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:53:46,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:53:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:53:54,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:53:54,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:53:54,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:53:55,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:00,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:05,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:09,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:09,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:09,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:09,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:54:10,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:15,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:20,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:24,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:24,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:24,357 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:26,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:31,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:33,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_cf1d81e3b206f0eeddf8.png with policy now +2023-07-12 07:54:33,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_e42387963e6467795ac3.png with policy now +2023-07-12 07:54:33,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,052 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_a24606b4de934e6b61c1.png with policy now +2023-07-12 07:54:33,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:54:33,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1880_19c60f744dd634323456.png with policy now +2023-07-12 07:54:33,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:54:33,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:54:33,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:54:33,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:54:33,381 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/57xkdxgj-media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 07:54:33,394 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g27zamfh-media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 07:54:33,395 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vgvbgnoe-media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 07:54:33,577 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nw35bl6y-media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 07:54:34,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:54:34,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 07:54:34,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 07:54:34,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 07:54:34,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 07:54:34,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:54:35,312 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zdaihrrq-media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 07:54:35,362 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0nkvj53q-media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 07:54:35,367 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4b5ldon-media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 07:54:35,402 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6tgh2zir-media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 07:54:36,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:39,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:39,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:39,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:39,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:54:41,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:42,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:54:44,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:54:46,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:50,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:54:52,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:54,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:54:54,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:54:54,029 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:54:54,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:54:54,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:54:54,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:54:54,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:54:58,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:54:58,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:54:58,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:03,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:03,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:03,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:03,741 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:03,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:06,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:06,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:09,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:09,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:55:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:10,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:13,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:13,244 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:13,244 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:13,246 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:15,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:15,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:20,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:23,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:24,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:24,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:24,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:24,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:25,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:25,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:25,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:26,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:31,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:31,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:31,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:34,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:34,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:34,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:34,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:37,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:39,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:39,207 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:39,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:55:40,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:40,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:40,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:43,559 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:45,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:45,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:45,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:45,015 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:47,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:47,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:55:48,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:54,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:55:55,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:55:55,075 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:55:55,075 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:55:55,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:55:55,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:55:55,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:55:55,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:55:55,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:55:55,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:00,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:03,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:04,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:56:04,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:56:04,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:56:04,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:56:05,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:09,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:56:09,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:56:09,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:56:09,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:56:09,641 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:56:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:10,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:11,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:56:11,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:11,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:16,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:19,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:56:21,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:25,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:25,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:25,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:27,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:32,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:37,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:39,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:56:40,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:40,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:40,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:42,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:47,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:52,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:56:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:56:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:56:55,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:56:58,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:03,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:04,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_5fabc2eb5b38fc6c441f.png with policy now +2023-07-12 07:57:04,267 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,267 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_362eab872b8e6854bfcb.png with policy now +2023-07-12 07:57:04,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,277 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_abe953372839c2e35d11.png with policy now +2023-07-12 07:57:04,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:57:04,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1890_61843d74c0b1b4039c5a.png with policy now +2023-07-12 07:57:04,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:04,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:04,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:04,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:04,550 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vs54hd9o-media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 07:57:04,557 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gx3sul2d-media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 07:57:04,574 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w6rqmfhn-media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 07:57:04,574 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8057oeb7-media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 07:57:07,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:07,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 07:57:07,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 07:57:07,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 07:57:07,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 07:57:07,520 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:57:07,859 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/odann1ub-media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 07:57:07,860 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9vpluri5-media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 07:57:07,867 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2wol7bz2-media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 07:57:07,871 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i7mxby7b-media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 07:57:09,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:09,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:57:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:10,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:14,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:14,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:15,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:20,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:23,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:23,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:23,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:23,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:23,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:25,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:25,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:25,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:25,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:30,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:31,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:31,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:33,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:33,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:33,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:33,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:35,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:39,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:39,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:39,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:57:40,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:40,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:40,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:40,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:43,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:43,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:43,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:43,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:46,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:47,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:47,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:51,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:57:54,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:57:54,514 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:57:54,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:57:54,516 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:57:55,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:57:55,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:57:55,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:57:55,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:57:55,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:57:57,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:02,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:03,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:05,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:05,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:05,180 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:05,182 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:08,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:09,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:58:10,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:10,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:10,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:11,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:11,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:14,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:14,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:14,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:14,978 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:14,980 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:19,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:19,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:19,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:25,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:25,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:25,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:25,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:25,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:25,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:25,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:25,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:27,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:27,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:31,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:35,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:35,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:35,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:35,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:35,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:35,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:37,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:39,581 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:58:39,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:58:39,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:58:39,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:58:39,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:58:40,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:40,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:40,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:42,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:43,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:58:43,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:58:47,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:52,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:58:55,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:58:55,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:58:55,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:58:57,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:02,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:07,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:09,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:59:10,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:10,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:10,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:12,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:17,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:22,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:25,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:25,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:25,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:27,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:32,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:34,310 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,311 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_755c75927be8fb0bccfb.png with policy now +2023-07-12 07:59:34,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_7ac136e8ccfb72e61b40.png with policy now +2023-07-12 07:59:34,330 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,331 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_e8722f8bf9b351a33b0d.png with policy now +2023-07-12 07:59:34,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 07:59:34,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1900_1895be2b314f5c3cc922.png with policy now +2023-07-12 07:59:34,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:59:34,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:59:34,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:59:34,385 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:59:34,659 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qa1zdyls-media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 07:59:34,661 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpp3wrnq-media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 07:59:34,674 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h36y1t5f-media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 07:59:34,693 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e9kncinm-media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 07:59:38,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:39,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 07:59:40,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:59:40,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:59:40,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 07:59:40,129 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 07:59:40,130 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 07:59:40,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 07:59:40,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 07:59:40,540 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zvb8vy5h-media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 07:59:40,546 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fkm40qm2-media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 07:59:40,561 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9zmd740-media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 07:59:40,580 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fku2fgn6-media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 07:59:40,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:40,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:40,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:43,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:43,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:59:48,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 07:59:49,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:53,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 07:59:53,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 07:59:53,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 07:59:53,164 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 07:59:55,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 07:59:55,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 07:59:55,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 07:59:55,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 07:59:56,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 07:59:56,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:00,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:02,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:02,564 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:02,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:02,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:04,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:04,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:06,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:09,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:00:10,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:00:10,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:10,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:11,715 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:12,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:12,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:12,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:12,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:12,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:17,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:20,203 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:20,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:23,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:23,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:23,445 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:23,445 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:23,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:25,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:00:25,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:25,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:28,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:28,218 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:29,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:33,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:33,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:33,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:33,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:34,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:36,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:36,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:39,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:00:40,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:40,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:40,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:43,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:43,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:43,620 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:43,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:44,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:00:44,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:45,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:50,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:00:52,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:00:54,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:00:55,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:00:55,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:00:55,002 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:00:55,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:00:55,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:00:56,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:00,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:01:00,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:01:02,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:05,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:01:05,425 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:01:05,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:01:05,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:01:07,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:08,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:01:08,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:01:09,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:01:10,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:01:10,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:01:10,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:01:10,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:01:10,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:10,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:12,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:16,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:01:16,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:01:17,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:22,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:25,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:01:25,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:25,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:27,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:32,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:37,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:39,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:01:40,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:01:40,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:40,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:43,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:48,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:53,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:01:55,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:01:55,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:01:55,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:01:59,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:04,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_6fd922b416ffd009954a.png with policy now +2023-07-12 08:02:04,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_5760a804d5e3912bf871.png with policy now +2023-07-12 08:02:04,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,733 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_34349e41d8bc032e1c7d.png with policy now +2023-07-12 08:02:04,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:02:04,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1910_f7431e028249d68b3d9b.png with policy now +2023-07-12 08:02:04,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:04,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:04,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:04,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:05,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:07,696 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6r27yrl0-media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 08:02:07,815 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wdmtgtar-media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 08:02:09,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:02:10,529 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zrpuc7z1-media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 08:02:10,661 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rbsqecva-media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 08:02:10,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:10,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:10,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:10,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 08:02:12,724 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 08:02:12,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 08:02:12,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 08:02:12,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:02:13,075 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/giwccv5d-media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 08:02:13,077 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c1mhn3g8-media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 08:02:13,083 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vhzs1dx5-media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 08:02:13,105 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/76gvkkqt-media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 08:02:15,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:16,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:20,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:22,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:25,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:25,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:25,521 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:25,523 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:25,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:25,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:25,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:27,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:28,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:28,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:33,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:34,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:34,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:34,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:34,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:36,771 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:36,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:38,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:39,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:02:40,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:40,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:40,854 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:44,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:44,785 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:45,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:45,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:45,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:45,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:02:49,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:52,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:02:52,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:02:55,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:02:55,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:02:55,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:02:55,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:02:56,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:02:56,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:02:56,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:02:56,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:00,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:00,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:00,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:05,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:05,924 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:05,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:05,925 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:05,927 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:08,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:08,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:09,591 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:03:10,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:10,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:10,853 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:10,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:16,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:16,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:16,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:16,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:16,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:17,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:17,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:21,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:25,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:26,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:26,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:27,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:27,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:27,160 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:27,162 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:31,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:33,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:33,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:37,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:37,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:37,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:37,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:37,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:39,592 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:03:41,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:41,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:41,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:41,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:03:41,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:03:41,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:03:41,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:03:42,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:47,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:49,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:03:49,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:03:52,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:03:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:03:56,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:03:56,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:03:57,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:02,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:07,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:09,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:04:11,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:11,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:11,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:13,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:18,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:23,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:26,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:26,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:28,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:33,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:36,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_0c56e18802b80a5e1cdb.png with policy now +2023-07-12 08:04:36,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_7d921cd7daa3b8f8aae7.png with policy now +2023-07-12 08:04:36,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,353 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_60d58608f7bfc9ddde76.png with policy now +2023-07-12 08:04:36,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:04:36,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1920_3e7c1f6543b9b9815c1e.png with policy now +2023-07-12 08:04:36,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:04:36,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:04:36,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:04:36,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:04:36,809 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j8e2dk62-media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 08:04:36,813 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r057h9bw-media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 08:04:36,817 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wac6l1b5-media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 08:04:36,818 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hltbak8g-media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 08:04:37,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:04:37,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 08:04:37,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 08:04:37,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 08:04:37,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 08:04:37,337 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:04:37,781 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9jmiw89h-media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 08:04:37,800 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gezev9am-media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 08:04:37,810 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cozq7609-media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 08:04:37,814 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/494own9a-media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 08:04:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:39,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:04:41,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:41,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:41,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:44,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:45,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:04:46,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:04:50,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:53,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:04:56,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:04:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:04:56,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:04:56,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:04:57,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:04:57,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:04:57,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:04:57,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:01,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:01,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:01,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:06,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:06,752 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:06,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:06,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:06,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:09,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:09,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:09,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:05:11,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:11,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:11,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:11,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:16,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:16,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:16,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:16,416 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:17,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:17,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:17,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:23,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:25,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:26,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:26,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:26,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:26,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:26,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:26,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:26,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:28,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:33,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:33,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:34,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:36,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:36,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:36,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:36,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:39,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:39,595 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:05:41,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:41,625 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:41,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:41,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:41,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:44,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:47,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:47,381 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:47,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:47,383 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:05:49,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:05:49,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:49,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:55,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:05:56,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:05:56,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:05:56,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:05:57,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:05:57,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:05:57,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:05:57,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:05:57,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:06:00,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:05,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:06:05,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:06:06,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:08,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:06:08,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:06:08,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:06:08,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:06:09,596 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:06:11,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:11,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:11,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:11,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:13,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:06:13,482 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:06:13,483 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:06:13,485 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:06:13,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:06:13,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:06:17,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:21,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:06:22,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:26,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:26,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:26,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:27,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:32,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:37,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:39,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:06:41,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:41,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:41,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:42,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:47,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:52,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:06:56,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:06:56,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:06:56,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:06:58,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:03,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:07,714 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,715 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_48056833d0b2d7f2157c.png with policy now +2023-07-12 08:07:07,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_dc6aa83bfa1de0638963.png with policy now +2023-07-12 08:07:07,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_8bf913d97ecc12b8092b.png with policy now +2023-07-12 08:07:07,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:07:07,751 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1930_cbdfa1d5c7889c908498.png with policy now +2023-07-12 08:07:07,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:07,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:07,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:07,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:08,070 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5p2jy24k-media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 08:07:08,076 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/syjjzg8r-media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 08:07:08,100 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8i4sb78p-media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 08:07:08,178 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rxm04c7j-media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 08:07:09,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:07:09,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:09,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 08:07:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 08:07:09,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 08:07:09,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 08:07:09,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:07:10,397 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g63bbsed-media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 08:07:10,495 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p46ibllb-media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 08:07:10,518 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n46ri2nu-media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 08:07:10,568 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/65gh3t8l-media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 08:07:12,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:12,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:12,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:14,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:17,959 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:17,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:19,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:25,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:25,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:27,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:27,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:27,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:27,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:27,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:27,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:27,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:30,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:33,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:33,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:35,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:37,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:37,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:37,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:37,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:39,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:07:41,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:42,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:42,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:42,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:42,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:42,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:46,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:48,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:48,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:48,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:48,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:07:50,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:07:50,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:52,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:57,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:07:58,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:07:58,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:07:58,026 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:07:58,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:07:58,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:07:58,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:07:58,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:07:58,592 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:03,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:06,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:06,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:08,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:08,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:08,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:08,923 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:08,925 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:09,602 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:08:13,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:13,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:13,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:08:14,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:14,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:14,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:19,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:19,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:19,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:19,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:19,240 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:22,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:22,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:24,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:28,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:28,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:28,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:08:28,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:28,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:28,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:28,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:29,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:30,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:30,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:34,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:38,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:38,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:38,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:38,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:38,646 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:39,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:08:40,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:43,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:08:43,393 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:08:43,393 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:08:43,395 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:08:43,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:43,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:43,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:08:46,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:46,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:08:46,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:08:51,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:56,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:08:58,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:08:58,686 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:08:58,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:01,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:06,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:09,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:09:12,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:13,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:13,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:13,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:17,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:22,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:27,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:28,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:28,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:28,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:32,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:37,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:38,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,139 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_55fed6437ec58a4992ec.png with policy now +2023-07-12 08:09:38,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_b7870fe1d164423912ca.png with policy now +2023-07-12 08:09:38,158 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_36d3d88a96d60bb7d69e.png with policy now +2023-07-12 08:09:38,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:09:38,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1940_42b89bc7a5b6ad127dd2.png with policy now +2023-07-12 08:09:38,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:09:38,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:09:38,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:09:38,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:09:38,565 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lkpgn2yi-media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 08:09:38,610 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9z4722yk-media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 08:09:38,614 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/26hbi3v9-media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 08:09:38,645 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1j3rzjoe-media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 08:09:39,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:09:42,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:09:42,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:09:42,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 08:09:42,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 08:09:42,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 08:09:42,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 08:09:42,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:09:42,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:43,065 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bvznvbhy-media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 08:09:43,078 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kfleqsdc-media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 08:09:43,087 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vinz16m4-media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 08:09:43,211 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u7lxxjj8-media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 08:09:43,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:43,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:43,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:47,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:09:48,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:50,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:09:54,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:09:56,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:09:56,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:09:56,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:09:56,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:09:58,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:09:58,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:09:58,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:09:58,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:09:58,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:09:59,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:05,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:06,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:06,520 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:06,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:06,522 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:06,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:06,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:09,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:10:11,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:13,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:13,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:13,709 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:14,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:16,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:16,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:16,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:16,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:16,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:21,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:22,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:22,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:26,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:27,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:27,339 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:27,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:27,341 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:29,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:29,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:29,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:30,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:30,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:31,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:37,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:37,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:37,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:37,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:37,907 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:38,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:38,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:39,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:10:42,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:44,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:44,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:44,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:46,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:48,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:49,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:49,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:49,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:49,079 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:53,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:10:54,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:10:54,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:10:58,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:10:58,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:10:58,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:10:58,602 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:10:59,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:10:59,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:10:59,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:10:59,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:02,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:11:02,967 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:11:05,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:09,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:11:09,127 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:11:09,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:11:09,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:11:09,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:11:10,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:11,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:11:11,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:11:13,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:11:13,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:11:13,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:11:13,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:11:14,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:14,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:14,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:11:16,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:19,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:11:19,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:11:21,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:26,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:29,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:29,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:29,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:11:32,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:37,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:39,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:11:42,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:44,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:44,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:44,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:11:48,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:53,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:58,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:11:59,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:11:59,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:11:59,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:03,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:08,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_2c29e26c44ad93206dd5.png with policy now +2023-07-12 08:12:08,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_a1d43d5ef09db63ec9f2.png with policy now +2023-07-12 08:12:08,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_a1baa48379524c1c8b17.png with policy now +2023-07-12 08:12:08,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:12:08,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1950_e23c84b831732e39304c.png with policy now +2023-07-12 08:12:08,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:08,634 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:08,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:08,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:09,048 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5gxbrbrh-media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 08:12:09,083 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3tj95eek-media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 08:12:09,103 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lfxk4m8z-media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 08:12:09,156 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mc42f72a-media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 08:12:09,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:12:09,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:14,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:12:14,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:12:14,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:14,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:15,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:15,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:15,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 08:12:15,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 08:12:15,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 08:12:15,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 08:12:15,322 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:12:15,875 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xf61teje-media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 08:12:15,905 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grmgs1hj-media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 08:12:15,988 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kizhbh83-media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 08:12:15,988 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yve46ed0-media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 08:12:18,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:20,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:23,335 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:25,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:27,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:27,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:27,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:27,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:29,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:12:29,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:12:29,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:31,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:31,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:31,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:36,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:38,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:38,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:38,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:38,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:39,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:39,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:39,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:12:42,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:45,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:12:45,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:12:45,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:12:47,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:48,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:48,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:48,586 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:48,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:48,588 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:12:53,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:55,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:12:55,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:12:58,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:12:59,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:12:59,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:12:59,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:12:59,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:01,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:01,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:01,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:03,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:03,406 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:04,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:09,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:09,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:13:10,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:10,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:10,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:10,140 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:11,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:14,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:17,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:17,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:17,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:19,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:19,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:20,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:20,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:20,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:20,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:25,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:27,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:27,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:29,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:29,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:29,920 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:29,922 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:30,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:32,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:32,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:32,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:35,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:35,641 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:36,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:39,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:13:40,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:40,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:40,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:40,098 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:42,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:43,656 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:43,657 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:44,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:13:44,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:13:44,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:13:44,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:13:47,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:13:47,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:13:47,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:13:47,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:51,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:13:51,672 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:13:52,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:13:57,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:02,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:02,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:03,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:08,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:09,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:14:13,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:17,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:17,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:14:17,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:19,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:24,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:29,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:32,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:14:32,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:32,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:34,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:39,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:39,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:14:39,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_e3afb0220a3fb4fd12f3.png with policy now +2023-07-12 08:14:39,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,923 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_35975d12907982332b52.png with policy now +2023-07-12 08:14:39,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_e84cb7c3e576b3bd6e96.png with policy now +2023-07-12 08:14:39,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:14:39,944 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1960_e96fba72991bb49f94d6.png with policy now +2023-07-12 08:14:39,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:14:39,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:14:39,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:14:39,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:14:40,227 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iyv8179n-media/images/validation_1960_35975d12907982332b52.png +2023-07-12 08:14:40,243 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lsq0rm8g-media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 08:14:40,260 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rfq4d9jh-media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 08:14:40,294 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c29pgtdo-media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 08:14:44,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:47,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:14:47,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:14:47,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:14:47,941 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:14:47,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:14:47,942 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 08:14:47,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 08:14:47,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_35975d12907982332b52.png +2023-07-12 08:14:47,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 08:14:47,944 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:14:48,284 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2yslq4i3-media/images/validation_1960_35975d12907982332b52.png +2023-07-12 08:14:48,329 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/655fjxj5-media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 08:14:49,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:14:49,291 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2hqrfp1j-media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 08:14:49,524 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sl4nhntb-media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 08:14:50,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:55,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:14:55,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:14:59,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:14:59,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:14:59,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:14:59,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:01,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:02,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:02,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:02,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:03,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:03,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:07,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:09,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:09,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:09,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:09,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:09,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:15:11,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:11,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:12,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:17,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:17,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:17,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:18,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:20,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:20,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:20,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:20,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:20,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:23,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:28,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:28,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:29,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:31,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:31,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:31,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:31,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:32,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:32,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:32,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:34,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:36,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:36,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:39,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:39,619 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:15:41,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:41,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:41,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:41,339 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:44,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:44,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:44,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:47,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:15:47,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:15:47,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:15:50,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:15:50,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:15:50,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:15:50,926 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:15:50,928 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:15:52,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:15:52,238 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:15:55,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:00,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:00,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:16:00,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:00,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:16:00,777 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:16:00,779 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:16:03,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:03,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:03,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:05,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:08,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:16:08,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:09,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:16:11,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:16:11,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:11,465 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:16:11,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:16:11,467 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:16:16,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:16:16,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:16:16,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:16:16,240 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:16:16,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:16:16,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:16,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:18,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:18,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:18,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:22,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:24,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:16:27,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:32,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:33,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:33,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:38,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:39,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:16:43,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:48,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:16:48,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:16:48,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:16:49,305 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:54,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:16:59,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:03,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:17:03,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:03,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:04,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:09,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:09,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:17:10,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_1c5cd897c6da1cf72c60.png with policy now +2023-07-12 08:17:10,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_b693a85e0ba7b021e8b5.png with policy now +2023-07-12 08:17:10,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,949 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_2418660f78d4aba7393a.png with policy now +2023-07-12 08:17:10,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:17:10,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1970_f777a51b498e8fce06d9.png with policy now +2023-07-12 08:17:11,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:11,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:11,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:11,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:11,252 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/38ay01bz-media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 08:17:11,274 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5n91va6k-media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 08:17:11,301 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dxe2v6sd-media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 08:17:11,307 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l3lah2fq-media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 08:17:12,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:12,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 08:17:12,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 08:17:12,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 08:17:12,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 08:17:12,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:17:12,989 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wwehsu26-media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 08:17:12,989 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/idf84ram-media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 08:17:12,998 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ymylitfh-media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 08:17:13,005 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9h5po8xk-media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 08:17:15,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:18,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:17:18,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:18,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:20,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:20,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:20,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:26,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:28,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:29,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:29,832 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:29,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:29,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:31,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:33,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:33,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:17:33,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:36,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:36,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:36,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:39,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:39,612 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:39,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:39,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:39,623 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:17:42,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:44,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:44,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:47,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:48,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:17:48,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:17:50,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:50,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:50,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:50,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:17:52,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:52,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:17:52,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:17:58,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:17:59,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:17:59,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:17:59,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:17:59,777 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:00,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:00,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:04,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:04,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:04,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:04,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:08,662 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:09,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:09,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:09,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:09,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:09,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:09,625 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:18:14,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:16,850 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:16,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:19,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:19,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:19,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:19,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:19,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:19,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:19,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:19,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:24,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:24,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:25,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:30,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:31,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:31,311 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:31,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:31,313 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:32,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:32,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:34,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:34,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:34,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:36,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:39,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:18:40,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:41,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:41,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:41,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:41,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:41,725 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:46,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:46,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:18:46,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:18:46,814 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:18:46,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:18:48,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:18:48,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:18:49,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:18:49,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:18:49,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:18:52,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:18:57,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:02,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:04,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:04,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:04,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:07,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:09,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:19:12,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:17,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:19,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:19,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:19,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:23,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:28,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:33,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:34,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:34,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:34,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:38,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:39,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:19:41,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_1db1fd51970584b5daa1.png with policy now +2023-07-12 08:19:41,429 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,429 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_7a8ef7854cf8e37f9cf7.png with policy now +2023-07-12 08:19:41,438 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_8caa5ac810c363d03b6e.png with policy now +2023-07-12 08:19:41,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:19:41,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1980_647b73db1b8d573d85d7.png with policy now +2023-07-12 08:19:41,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:19:41,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:19:41,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:19:41,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:19:41,738 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/66acku56-media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 08:19:41,739 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ny344vyy-media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 08:19:41,775 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xfef5vle-media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 08:19:41,842 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w2c4qdv9-media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 08:19:44,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:45,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:19:45,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:19:45,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 08:19:45,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 08:19:45,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 08:19:45,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 08:19:45,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:19:45,588 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q92b4hie-media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 08:19:45,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8w3ab78-media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 08:19:45,626 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/huj5byyo-media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 08:19:45,631 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fsdjr60a-media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 08:19:49,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:19:49,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:19:49,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:19:49,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:19:51,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:19:53,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:19:54,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:00,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:00,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:00,700 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:00,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:00,702 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:01,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:01,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:05,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:05,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:05,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:05,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:09,215 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:09,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:20:10,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:10,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:10,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:10,775 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:10,777 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:15,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:17,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:17,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:20,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:20,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:20,413 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:20,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:20,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:20,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:21,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:21,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:25,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:25,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:26,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:30,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:30,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:30,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:30,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:31,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:33,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:33,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:35,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:35,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:35,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:37,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:39,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:20:41,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:41,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:41,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:41,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:41,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:42,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:48,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:49,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:49,471 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:50,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:20:50,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:20:50,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:20:51,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:20:51,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:20:51,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:20:51,455 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:20:53,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:20:57,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:20:57,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:20:59,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:02,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:21:02,232 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:21:02,232 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:21:02,234 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:21:04,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:05,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:21:05,502 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:21:05,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:05,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:05,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:09,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:21:09,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:12,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:21:12,519 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:21:12,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:21:12,521 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:21:13,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:21:13,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:21:15,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:17,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:21:17,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:21:17,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:21:17,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:21:21,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:21,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:21,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:21,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:21,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:21:21,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:21:26,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:31,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:36,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:36,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:36,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:37,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:39,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:21:42,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:47,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:51,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:21:51,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:21:51,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:21:53,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:21:58,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:03,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:06,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:22:06,309 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:06,309 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:08,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:09,634 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:22:11,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_e53b48fcc568a502f437.png with policy now +2023-07-12 08:22:11,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_7e8731b3e3747b303dfe.png with policy now +2023-07-12 08:22:11,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,839 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_283a622f33c91b0b5af0.png with policy now +2023-07-12 08:22:11,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:22:11,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_1990_35002cfdf6c8ccfffdcd.png with policy now +2023-07-12 08:22:11,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:11,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:11,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:11,910 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:12,141 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f8jrn5lx-media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 08:22:12,192 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lee429j9-media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 08:22:12,194 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eslo6fwy-media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 08:22:12,202 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wu9nbcin-media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 08:22:13,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:17,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:17,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:17,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 08:22:17,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 08:22:17,815 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 08:22:17,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 08:22:17,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:22:18,148 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q1gyizwa-media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 08:22:18,172 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z4i7bavi-media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 08:22:18,197 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/svvj5qnp-media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 08:22:18,204 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xc7ayvfk-media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 08:22:19,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:21,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:22:21,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:21,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:21,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:24,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:25,830 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:30,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:30,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:30,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:30,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:30,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:34,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:34,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:35,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:36,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:22:36,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:36,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:39,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:22:40,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:40,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:40,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:40,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:40,791 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:43,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:43,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:46,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:50,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:22:50,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:22:50,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:22:50,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:22:51,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:22:51,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:22:52,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:22:52,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:22:52,874 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:22:57,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:01,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:01,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:01,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:01,114 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:01,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:01,888 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:03,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:06,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:06,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:08,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:09,637 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:23:10,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:11,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:11,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:11,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:11,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:13,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:18,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:20,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:20,104 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:21,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:23:21,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:21,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:22,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:22,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:22,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:22,164 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:24,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:29,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:29,118 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:29,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:32,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:32,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:32,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:32,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:35,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:37,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:23:37,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:37,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:38,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:38,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:39,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:23:41,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:43,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:43,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:43,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:43,189 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:46,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:47,148 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:47,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:47,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:23:47,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:23:47,804 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:23:47,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:23:52,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:23:52,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:23:52,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:23:52,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:23:56,164 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:23:56,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:23:57,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:02,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:07,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:07,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:07,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:07,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:09,639 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:24:13,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:18,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:22,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:22,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:22,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:23,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:28,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:33,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:37,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:37,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:37,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:38,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:39,640 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:24:42,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,523 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_1f7c7153c03240078b4d.png with policy now +2023-07-12 08:24:42,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_56c478641cd971b63c78.png with policy now +2023-07-12 08:24:42,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_0669630fb67332c6ded5.png with policy now +2023-07-12 08:24:42,556 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:24:42,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2000_81cc3487bd6c41e95ef1.png with policy now +2023-07-12 08:24:42,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:24:42,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:24:42,599 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:24:42,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:24:42,830 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dugt5132-media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 08:24:42,866 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lqgmk4cc-media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 08:24:42,873 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/732jlvmw-media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 08:24:42,909 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c822fzfl-media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 08:24:44,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:50,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:50,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:24:50,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:24:50,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 08:24:50,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 08:24:50,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 08:24:50,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 08:24:50,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:24:50,769 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8bojs5u-media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 08:24:50,791 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/amh3kfln-media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 08:24:50,791 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5rd3t4r4-media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 08:24:50,798 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9n8gvit-media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 08:24:52,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:24:52,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:24:52,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:24:52,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:24:55,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:24:59,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:01,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:01,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:01,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:01,431 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:01,433 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:07,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:08,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:08,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:08,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:08,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:08,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:09,641 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:25:11,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:11,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:11,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:11,209 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:13,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:17,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:17,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:18,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:21,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:21,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:21,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:21,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:23,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:23,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:23,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:23,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:26,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:26,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:28,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:31,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:31,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:31,994 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:31,996 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:33,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:35,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:35,497 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:38,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:38,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:38,617 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:39,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:25:39,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:42,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:42,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:42,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:42,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:44,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:44,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:44,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:49,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:25:52,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:25:52,558 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:25:52,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:25:52,560 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:25:53,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:25:53,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:25:53,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:25:53,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:25:53,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:25:55,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:00,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:01,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:26:01,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:26:01,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:26:01,882 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:26:02,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:26:02,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:26:06,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:08,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:08,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:08,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:10,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:26:11,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:26:12,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:26:12,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:12,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:26:12,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:26:12,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:26:17,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:26:17,395 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:26:17,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:26:17,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:26:17,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:20,744 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:26:20,745 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:26:23,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:23,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:23,976 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:23,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:29,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:34,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:39,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:39,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:39,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:39,080 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:39,645 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:26:44,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:49,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:26:54,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:26:54,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:26:54,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:26:55,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:00,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:05,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:09,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:09,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:09,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:09,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:27:10,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:12,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,202 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_ebb2fde791af714a87fa.png with policy now +2023-07-12 08:27:12,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_fe06a780b0eeb017b688.png with policy now +2023-07-12 08:27:12,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_3246141969b6e42324be.png with policy now +2023-07-12 08:27:12,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:27:12,238 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2010_3fcb388422ea2886399f.png with policy now +2023-07-12 08:27:12,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:12,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:12,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:12,286 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:12,571 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1egk7yr3-media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 08:27:12,572 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v1co0t3j-media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 08:27:12,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ics6e0n5-media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 08:27:12,619 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sza7a92h-media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 08:27:15,011 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:27:15,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 08:27:15,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 08:27:15,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 08:27:15,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 08:27:15,013 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:27:15,397 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zeypwevo-media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 08:27:15,424 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mslp45cx-media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 08:27:15,451 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jftzn453-media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 08:27:15,459 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uk9ctn8c-media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 08:27:16,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:21,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:22,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:24,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:24,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:24,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:24,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:27,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:32,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:32,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:32,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:32,413 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:32,415 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:27:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:38,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:39,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:27:39,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:39,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:39,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:42,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:42,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:42,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:42,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:42,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:43,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:49,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:51,171 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:27:51,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:27:53,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:27:53,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:27:53,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:27:53,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:27:54,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:27:55,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:27:55,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:27:55,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:27:59,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:00,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:00,186 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:02,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:02,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:02,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:03,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:05,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:09,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:09,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:09,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:28:10,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:10,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:10,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:28:10,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:13,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:13,517 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:13,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:13,519 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:16,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:18,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:18,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:22,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:24,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:24,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:24,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:24,083 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:25,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:25,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:25,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:28:27,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:27,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:28,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:33,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:34,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:34,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:34,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:34,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:36,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:36,407 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:39,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:39,652 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:28:41,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:41,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:41,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:28:44,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:44,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:44,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:44,866 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:44,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:45,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:45,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:49,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:28:49,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:28:49,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:28:49,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:28:50,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:54,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:28:54,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:28:55,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:28:56,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:28:56,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:28:56,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:00,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:05,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:09,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:29:10,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:11,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:11,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:11,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:16,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:21,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:26,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:26,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:26,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:27,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:32,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:37,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:39,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:29:41,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:41,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:41,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:42,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:43,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_13af1ccf733709899b51.png with policy now +2023-07-12 08:29:43,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,886 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_6f7e3605df1d5a3bcd5d.png with policy now +2023-07-12 08:29:43,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,895 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_ae08a04cbc13fea1f1e4.png with policy now +2023-07-12 08:29:43,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:29:43,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2020_05878893911aa97a5d5b.png with policy now +2023-07-12 08:29:43,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:29:43,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:29:43,947 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:29:43,951 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:29:44,188 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qqc5yul4-media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 08:29:44,202 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gq37ij9o-media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 08:29:44,232 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d2lztrf7-media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 08:29:44,274 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wh4b72jw-media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 08:29:48,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:48,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:29:48,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:29:48,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 08:29:48,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 08:29:48,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 08:29:48,705 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 08:29:48,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:29:49,073 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zmouv3nn-media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 08:29:49,086 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9n0fqzl4-media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 08:29:49,095 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27i27fau-media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 08:29:49,279 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z61ewems-media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 08:29:53,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:29:54,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:29:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:29:56,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:29:56,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:29:57,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:29:59,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:03,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:03,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:03,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:03,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:05,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:06,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:06,730 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:09,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:30:10,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:12,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:30:12,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:12,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:13,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:13,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:13,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:13,293 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:15,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:15,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:15,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:21,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:22,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:22,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:22,985 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:22,987 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:24,761 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:24,762 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:26,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:27,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:27,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:30:27,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:31,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:33,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:33,474 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:33,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:33,476 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:33,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:33,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:36,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:39,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:30:42,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:42,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:42,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:30:42,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:42,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:43,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:43,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:43,014 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:43,016 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:47,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:51,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:30:51,985 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:30:52,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:30:52,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:30:52,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:30:52,804 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:30:52,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:30:57,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:30:57,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:30:58,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:01,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:31:01,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:03,358 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:03,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:31:03,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:31:03,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:31:03,550 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:31:08,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:09,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:31:10,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:31:10,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:13,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:31:13,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:13,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:31:13,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:31:13,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:13,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:31:13,870 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:31:13,871 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:31:18,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:31:18,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:31:18,475 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:31:18,477 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:31:18,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:19,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:31:19,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:24,079 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:28,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:31:28,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:28,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:31:29,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:34,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:39,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:39,658 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:31:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:31:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:43,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:31:44,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:49,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:54,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:31:58,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:31:58,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:31:58,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:00,417 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:05,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:09,659 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:32:10,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:12,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,389 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_1821049762b6b5d37c08.png with policy now +2023-07-12 08:32:12,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_e34ade0521fc9adbaae3.png with policy now +2023-07-12 08:32:12,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_d0209e06d5c81f80ec0d.png with policy now +2023-07-12 08:32:12,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:32:12,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2030_835fcd4e7ec0584630ce.png with policy now +2023-07-12 08:32:12,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:12,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:12,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:12,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:12,705 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/efnvb5db-media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 08:32:12,732 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3frk66le-media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 08:32:12,746 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5b8qkbk3-media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 08:32:12,787 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7fwfjl7i-media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 08:32:13,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:13,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 08:32:13,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 08:32:13,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 08:32:13,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 08:32:13,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:32:13,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:13,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:13,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:13,643 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/95tbbep1-media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 08:32:13,651 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t2xhbjc5-media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 08:32:13,659 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/io0906j1-media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 08:32:13,684 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jv2zq1pt-media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 08:32:16,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:21,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:22,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:23,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:26,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:28,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:28,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:28,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:31,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:31,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:32,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:32,601 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:32,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:32,604 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:36,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:39,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:32:40,364 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:40,365 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:41,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:42,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:42,425 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:42,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:42,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:43,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:43,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:32:46,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:49,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:49,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:51,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:32:51,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:32:51,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:32:51,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:32:52,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:58,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:32:58,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:32:58,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:32:59,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:32:59,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:32:59,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:02,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:02,608 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:02,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:02,610 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:03,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:07,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:07,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:09,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:09,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:33:13,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:13,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:13,513 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:13,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:14,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:14,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:14,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:14,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:16,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:16,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:20,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:23,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:23,574 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:23,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:23,576 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:25,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:25,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:25,624 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:29,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:29,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:29,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:30,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:33,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:33,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:33,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:33,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:34,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:35,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:39,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:33:41,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:43,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:44,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:44,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:44,029 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:44,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:44,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:44,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:33:46,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:48,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:33:48,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:33:48,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:33:48,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:33:51,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:52,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:33:52,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:33:56,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:33:59,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:33:59,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:33:59,558 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:02,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:07,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:09,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:34:12,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:14,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:14,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:14,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:17,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:22,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:27,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:29,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:29,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:29,726 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:33,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:38,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:39,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:34:43,253 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,253 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_6a4be91a47a6698809a9.png with policy now +2023-07-12 08:34:43,262 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,262 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_167355b47a08bf8109e9.png with policy now +2023-07-12 08:34:43,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,272 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_7914a4c603a50f2ea240.png with policy now +2023-07-12 08:34:43,282 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:34:43,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2040_6b091a76f28e18553ef9.png with policy now +2023-07-12 08:34:43,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:34:43,323 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:34:43,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:34:43,325 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:34:43,613 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c92ef3gt-media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 08:34:43,638 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tayfn3cy-media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 08:34:43,640 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l5d5njmc-media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 08:34:43,641 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pyohl9t1-media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 08:34:44,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:44,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:44,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:44,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:34:46,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:34:46,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:34:46,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 08:34:46,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 08:34:46,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 08:34:46,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 08:34:46,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:34:47,326 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wjov2f75-media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 08:34:47,360 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mlrjr6hn-media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 08:34:47,376 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oe744zs2-media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 08:34:47,385 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27dyyobl-media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 08:34:49,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:53,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:34:55,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:34:55,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:34:59,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:34:59,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:34:59,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:00,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:03,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:03,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:03,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:03,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:04,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:04,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:05,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:09,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:35:11,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:12,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:12,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:12,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:12,682 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:13,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:13,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:14,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:14,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:14,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:16,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:22,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:22,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:22,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:22,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:22,135 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:22,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:22,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:27,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:29,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:29,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:29,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:31,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:31,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:31,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:31,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:31,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:31,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:32,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:38,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:39,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:35:41,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:41,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:41,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:41,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:41,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:44,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:44,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:44,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:44,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:35:49,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:50,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:50,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:51,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:35:51,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:35:51,927 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:35:51,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:35:55,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:35:59,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:35:59,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:35:59,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:35:59,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:35:59,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:00,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:01,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:36:01,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:36:01,753 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:36:01,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:36:06,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:08,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:36:08,216 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:36:09,668 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:36:11,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:12,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:36:12,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:36:12,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:36:12,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:36:14,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:36:14,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:36:14,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:17,231 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:36:17,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:36:17,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:17,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:36:17,496 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:36:17,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:36:17,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:36:23,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:26,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:36:26,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:36:28,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:29,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:36:29,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:36:29,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:33,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:38,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:39,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:36:44,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:44,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:36:44,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:36:44,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:36:49,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:54,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:36:59,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:00,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:00,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:05,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:09,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:37:10,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:12,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,076 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_95ba8564905e15131c15.png with policy now +2023-07-12 08:37:12,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,086 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_025a92e01716e1b46d6d.png with policy now +2023-07-12 08:37:12,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_d4fb051468326f3eb57a.png with policy now +2023-07-12 08:37:12,106 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:37:12,106 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2050_8ea42d44ae0516e0ca5e.png with policy now +2023-07-12 08:37:12,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:12,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:12,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:12,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:12,403 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sgy05aqm-media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 08:37:12,410 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pb49mgdt-media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 08:37:12,412 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dc09qtzb-media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 08:37:12,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l5eksgej-media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 08:37:15,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:15,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:15,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:16,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:20,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:20,521 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:20,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 08:37:20,522 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 08:37:20,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 08:37:20,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 08:37:20,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:37:20,859 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i0egwudq-media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 08:37:20,919 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vrxnypbz-media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 08:37:20,919 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1z6bbz9-media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 08:37:20,959 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/viyihogi-media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 08:37:21,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:22,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:27,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:29,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:30,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:30,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:30,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:31,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:31,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:31,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:31,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:32,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:38,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:38,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:38,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:39,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:37:42,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:42,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:42,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:42,298 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:44,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:45,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:37:45,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:37:45,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:37:47,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:47,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:37:49,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:52,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:37:52,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:37:52,364 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:37:52,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:37:54,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:37:56,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:37:56,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:00,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:00,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:02,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:02,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:02,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:02,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:05,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:05,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:05,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:09,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:38:10,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:12,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:12,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:12,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:12,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:14,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:14,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:15,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:15,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:15,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:16,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:22,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:23,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:23,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:23,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:23,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:23,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:23,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:27,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:30,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:30,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:30,324 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:32,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:32,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:33,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:33,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:33,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:33,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:38,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:39,675 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:38:41,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:41,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:43,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:44,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:44,158 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:44,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:44,160 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:46,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:38:46,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:38:46,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:38:48,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:48,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:38:48,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:38:48,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:38:48,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:38:50,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:38:50,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:38:54,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:38:59,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:01,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:01,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:01,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:05,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:09,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:39:10,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:15,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:16,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:16,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:16,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:21,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:26,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:31,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:31,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:31,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:32,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:37,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:39,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:39:42,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:43,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_e0fa444f603a66029fbe.png with policy now +2023-07-12 08:39:43,481 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_d6f502d4c0d1394b0881.png with policy now +2023-07-12 08:39:43,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,492 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_ad32613282f23549a8bd.png with policy now +2023-07-12 08:39:43,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:39:43,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2060_5f6fadce63310a783572.png with policy now +2023-07-12 08:39:43,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:39:43,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:39:43,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:39:43,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:39:43,876 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hwgbq58c-media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 08:39:43,907 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ve5sww1b-media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 08:39:43,912 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/le6uc9w5-media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 08:39:44,056 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qw0drbd0-media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 08:39:45,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:39:45,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 08:39:45,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 08:39:45,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 08:39:45,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 08:39:45,136 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:39:45,506 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/85ycteza-media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 08:39:45,527 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/afd46fix-media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 08:39:45,528 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4q6q0ve1-media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 08:39:45,552 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/opyo5xo7-media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 08:39:46,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:39:46,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:39:46,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:39:48,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:54,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:39:54,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:39:54,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:00,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:01,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:01,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:01,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:03,165 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:03,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:03,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:03,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:03,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:05,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:09,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:40:11,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:12,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:12,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:13,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:13,035 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:13,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:13,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:17,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:18,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:18,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:18,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:21,197 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:21,198 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:22,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:23,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:23,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:23,586 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:23,587 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:28,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:30,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:30,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:33,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:33,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:33,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:33,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:34,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:34,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:34,060 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:34,062 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:38,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:39,228 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:39,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:39,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:40:44,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:44,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:44,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:44,026 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:44,028 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:48,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:48,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:40:48,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:40:48,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:40:48,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:40:49,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:55,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:40:55,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:40:55,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:40:55,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:40:55,118 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:40:57,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:40:57,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:00,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:03,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:03,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:03,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:05,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:41:05,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:41:05,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:41:05,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:41:06,047 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:06,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:41:06,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:09,680 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:41:11,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:15,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:15,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:41:15,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:41:15,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:41:15,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:41:17,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:18,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:18,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:18,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:20,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:41:20,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:41:20,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:41:20,505 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:41:23,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:24,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:41:24,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:41:28,336 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:33,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:33,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:33,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:33,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:38,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:39,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:41:43,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:48,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:41:48,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:41:48,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:41:49,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:54,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:41:59,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:03,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:03,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:03,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:05,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:09,682 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:42:10,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:14,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:14,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_19ed657f9c10dbd875ca.png with policy now +2023-07-12 08:42:15,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:15,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_e5f36a2a698bd96464b3.png with policy now +2023-07-12 08:42:15,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:15,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_cf0e421ac16cf53c2d5a.png with policy now +2023-07-12 08:42:15,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:42:15,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2070_df0b7fe6b5a5f888eeac.png with policy now +2023-07-12 08:42:15,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:15,062 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:15,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:15,064 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:15,357 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gujx60cg-media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 08:42:15,410 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b2sm0i5m-media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 08:42:15,456 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nnjs9jiv-media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 08:42:16,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:16,451 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r5orokgt-media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 08:42:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:18,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:18,787 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:42:18,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 08:42:18,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 08:42:18,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 08:42:18,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 08:42:18,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:42:19,150 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wtpuf8aw-media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 08:42:19,154 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5st5wuxv-media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 08:42:19,157 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vu717kbe-media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 08:42:19,177 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pjwbs952-media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 08:42:21,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:24,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:26,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:27,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:31,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:34,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:34,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:34,158 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:34,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:34,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:34,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:34,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:36,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:42:36,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:37,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:39,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:42:42,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:45,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:45,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:45,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:45,007 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:45,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:42:45,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:49,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:42:49,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:42:49,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:42:53,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:42:54,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:42:55,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:42:55,408 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:42:55,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:42:55,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:42:58,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:03,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:03,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:04,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:05,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:05,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:05,391 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:05,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:05,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:05,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:05,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:09,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:09,685 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:43:12,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:12,879 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:14,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:16,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:16,030 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:16,031 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:16,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:19,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:20,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:20,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:20,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:22,065 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:22,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:25,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:25,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:25,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:25,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:25,980 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:30,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:31,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:31,082 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:35,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:35,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:35,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:35,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:35,777 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:35,778 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:35,779 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:36,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:39,686 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:43:40,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:40,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:41,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:46,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:46,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:46,025 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:46,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:47,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:49,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:49,111 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:43:50,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:43:50,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:43:50,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:43:51,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:43:51,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:43:51,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:43:51,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:43:52,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:57,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:43:58,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:43:58,126 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:44:02,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:05,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:05,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:05,548 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:08,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:09,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:44:13,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:18,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:20,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:20,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:20,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:23,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:28,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:33,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:35,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:35,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:35,659 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:38,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:39,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:44:44,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:45,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_df994cef7f6357e3664e.png with policy now +2023-07-12 08:44:45,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_4c3033cd3536b3f44094.png with policy now +2023-07-12 08:44:45,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_0d50a284079de0683ad8.png with policy now +2023-07-12 08:44:45,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:44:45,545 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2080_7ed8ebb926937feea5e8.png with policy now +2023-07-12 08:44:45,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:44:45,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:44:45,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:44:45,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:44:45,839 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xvetz3gn-media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 08:44:45,844 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tkrytovb-media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 08:44:45,856 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0uo5se8h-media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 08:44:45,863 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ta9l84jk-media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 08:44:50,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:50,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:44:50,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:44:50,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:44:52,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:44:52,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:44:52,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 08:44:52,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 08:44:52,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 08:44:52,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 08:44:52,394 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:44:52,740 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1a9qrpf0-media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 08:44:52,742 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q8q6tlbw-media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 08:44:52,768 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y7jpnj0w-media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 08:44:52,782 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r9bzqi7h-media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 08:44:56,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:44:56,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:01,409 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:01,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:05,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:05,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:05,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:06,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:06,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:06,115 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:06,117 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:07,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:09,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:45:10,424 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:10,425 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:12,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:16,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:16,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:16,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:16,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:18,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:19,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:19,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:20,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:20,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:20,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:23,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:26,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:26,360 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:26,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:26,362 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:28,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:28,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:28,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:33,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:35,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:35,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:35,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:36,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:36,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:36,254 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:36,256 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:37,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:37,470 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:39,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:39,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:45:44,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:45,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:45,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:45,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:45,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:45:46,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:45:46,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:49,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:50,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:45:50,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:45:50,770 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:45:55,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:45:55,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:45:56,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:45:56,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:45:56,635 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:45:56,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:01,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:04,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:46:04,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:05,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:05,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:05,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:06,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:06,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:46:06,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:46:06,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:46:06,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:09,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:46:12,492 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:13,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:46:13,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:17,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:46:17,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:46:17,012 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:46:17,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:18,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:21,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:21,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:21,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:21,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:46:21,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:46:21,791 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:46:21,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:46:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:46:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:23,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:28,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:31,744 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:46:33,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:36,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:36,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:36,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:39,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:39,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:46:44,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:49,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:46:51,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:46:51,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:46:51,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:46:55,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:00,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:05,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:06,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:06,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:09,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:47:10,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:15,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:15,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_acf992ec2207b3ea2d60.png with policy now +2023-07-12 08:47:15,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_f680c2bff58b85913aa3.png with policy now +2023-07-12 08:47:15,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_b72a6b1b7e4871bb5d8f.png with policy now +2023-07-12 08:47:15,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:47:15,887 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2090_c267c69f2a522dc95745.png with policy now +2023-07-12 08:47:15,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:15,928 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:15,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:15,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:16,172 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0bane34p-media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 08:47:16,236 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqq9ul3r-media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 08:47:16,240 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x10ocwxk-media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 08:47:16,292 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9zsytd4-media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 08:47:16,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:47:16,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 08:47:16,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 08:47:16,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 08:47:16,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 08:47:17,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:47:17,356 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vlmyzwec-media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 08:47:17,370 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eakgf9ix-media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 08:47:17,375 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t765t8wp-media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 08:47:17,385 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mnpxswpq-media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 08:47:20,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:21,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:21,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:47:21,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:26,014 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:26,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:26,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:31,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:35,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:35,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:35,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:35,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:35,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:36,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:36,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:37,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:39,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:47:42,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:44,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:47:44,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:45,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:45,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:45,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:45,384 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:47,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:51,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:47:51,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:47:53,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:47:53,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:47:53,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:47:55,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:47:55,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:47:55,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:47:55,905 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:47:59,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:02,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:02,076 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:05,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:05,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:05,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:05,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:05,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:06,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:06,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:09,695 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:48:10,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:11,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:11,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:16,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:17,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:17,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:17,036 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:17,038 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:20,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:20,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:21,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:48:21,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:21,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:21,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:27,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:27,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:27,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:27,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:27,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:29,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:29,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:33,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:37,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:48:37,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:37,064 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:37,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:37,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:37,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:37,629 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:38,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:38,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:38,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:39,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:48:44,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:47,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:48:47,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:47,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:47,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:47,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:49,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:51,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:48:51,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:48:51,777 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:48:51,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:48:52,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:48:52,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:48:52,064 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:48:55,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:48:56,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:48:56,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:49:00,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:05,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:07,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:07,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:07,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:09,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:49:10,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:15,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:20,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:22,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:22,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:22,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:26,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:31,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:36,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:37,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:37,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:37,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:39,697 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:49:41,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:46,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,285 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_ed6b49b703996f58696a.png with policy now +2023-07-12 08:49:46,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_333e872b5de120d26892.png with policy now +2023-07-12 08:49:46,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_38040cf3cd9cbe1bc314.png with policy now +2023-07-12 08:49:46,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:49:46,316 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2100_2c125071cfc5a2fe04e6.png with policy now +2023-07-12 08:49:46,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:49:46,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:49:46,357 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:49:46,359 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:49:46,665 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nk3sofxw-media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 08:49:46,697 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/osat85ab-media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 08:49:46,715 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/488dm2sp-media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 08:49:46,717 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3zq85ocv-media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 08:49:47,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:50,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:49:50,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 08:49:50,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 08:49:50,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 08:49:50,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 08:49:50,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:49:51,022 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jo1s7lhu-media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 08:49:51,034 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hckcq32e-media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 08:49:51,046 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ck3jelf4-media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 08:49:51,047 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ltl0wgua-media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 08:49:52,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:49:52,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:49:52,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:49:53,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:56,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:49:58,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:49:59,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:03,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:06,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:06,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:06,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:06,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:08,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:50:08,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:08,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:08,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:08,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:09,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:50:09,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:15,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:16,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:16,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:16,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:16,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:17,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:17,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:21,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:23,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:23,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:50:23,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:26,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:27,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:27,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:27,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:27,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:27,222 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:32,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:35,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:35,697 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:37,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:37,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:37,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:37,712 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:37,714 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:38,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:38,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:39,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:50:43,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:44,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:44,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:47,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:47,557 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:47,557 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:47,559 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:50:48,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:53,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:50:53,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:50:53,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:50:53,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:50:54,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:50:58,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:50:58,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:50:58,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:50:58,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:00,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:02,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:02,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:05,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:08,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:08,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:08,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:51:08,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:51:08,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:51:08,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:09,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:51:10,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:11,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:11,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:15,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:19,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:51:19,027 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:51:19,027 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:51:19,029 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:20,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:20,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:21,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:23,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:23,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:23,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:51:23,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:51:23,742 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:51:23,744 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:51:26,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:29,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:51:29,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:51:31,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:36,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:38,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:38,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:51:38,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:39,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:51:41,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:46,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:51,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:51:53,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:51:53,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:51:53,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:51:57,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:02,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:07,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:08,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:08,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:08,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:09,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:52:12,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:17,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:18,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_62fae4a01ba8422d6ed9.png with policy now +2023-07-12 08:52:18,026 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,027 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_008df6c26e19a483defd.png with policy now +2023-07-12 08:52:18,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_3ef95d0764b88f6fbafa.png with policy now +2023-07-12 08:52:18,047 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:52:18,047 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2110_6a7c69b67225aaafd76d.png with policy now +2023-07-12 08:52:18,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:18,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:18,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:18,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:52:18,344 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a4x1847f-media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 08:52:18,348 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5lifjk4z-media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 08:52:18,367 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d6xrhlum-media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 08:52:18,369 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z75j56gn-media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 08:52:23,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:23,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:23,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:23,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 08:52:24,250 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 08:52:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 08:52:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 08:52:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:52:24,599 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p2darzwz-media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 08:52:24,614 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpophh2n-media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 08:52:24,648 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0a75aako-media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 08:52:24,653 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nm0sgz57-media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 08:52:27,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:28,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:33,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:34,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:37,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:37,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:37,335 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:37,336 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:52:39,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:39,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:39,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:39,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:39,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:52:42,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:52:42,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:44,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:47,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:47,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:47,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:47,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:52:50,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:51,297 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:52:51,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:52:54,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:52:54,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:52:54,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:52:56,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:52:57,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:52:57,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:52:57,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:52:57,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:00,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:00,314 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:01,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:07,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:07,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:07,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:07,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:07,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:09,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:09,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:09,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:09,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:09,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:09,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:53:12,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:17,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:17,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:17,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:17,698 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:17,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:18,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:18,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:22,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:25,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:25,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:25,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:27,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:27,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:28,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:28,259 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:28,259 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:28,262 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:32,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:36,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:36,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:38,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:38,302 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:38,303 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:38,304 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:38,306 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:39,707 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:53:40,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:40,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:40,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:43,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:45,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:45,578 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:48,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:48,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:48,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:48,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:48,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:52,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:53:52,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:53:52,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:53:52,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:53:53,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:53:54,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:53:54,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:53:55,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:53:55,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:53:55,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:53:59,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:04,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:09,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:09,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:54:10,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:10,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:10,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:14,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:19,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:24,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:25,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:25,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:25,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:29,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:34,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:39,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:39,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:54:40,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:40,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:40,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:44,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:47,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,155 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_b4413d53814dba669bcf.png with policy now +2023-07-12 08:54:47,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,165 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_7999661823c5d1a194ca.png with policy now +2023-07-12 08:54:47,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_1d395d84a1aa765d6c1b.png with policy now +2023-07-12 08:54:47,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:54:47,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2120_fe5c8ad23d0df9553dbf.png with policy now +2023-07-12 08:54:47,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:54:47,326 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:54:47,326 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:54:47,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:54:47,501 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/09x6xszy-media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 08:54:47,523 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2dz06vbk-media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 08:54:47,541 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iop8q1kb-media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 08:54:47,657 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ji45pzwh-media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 08:54:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:54:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 08:54:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 08:54:48,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 08:54:48,862 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 08:54:48,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:54:49,230 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gzfwq09l-media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 08:54:49,235 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1se9atxu-media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 08:54:49,253 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ne18v0qm-media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 08:54:49,253 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ry5wnhqo-media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 08:54:49,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:55,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:54:55,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:54:55,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:54:55,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:54:56,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:54:57,878 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:00,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:05,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:05,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:05,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:05,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:06,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:06,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:06,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:09,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:55:10,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:10,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:10,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:11,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:15,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:15,319 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:15,320 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:15,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:15,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:15,907 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:17,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:23,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:24,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:26,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:26,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:26,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:26,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:26,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:26,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:26,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:28,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:33,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:33,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:33,939 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:36,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:36,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:36,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:36,412 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:39,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:39,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:55:41,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:41,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:41,371 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:42,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:42,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:44,379 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:47,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:47,226 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:47,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:47,229 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:55:49,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:52,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:55:52,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:55:55,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:55:56,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:55:56,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:55:56,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:55:57,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:55:57,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:55:57,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:55:57,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:01,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:01,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:01,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:06,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:08,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:56:08,144 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:56:08,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:56:08,147 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:09,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:56:10,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:10,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:12,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:12,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:12,128 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:12,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:17,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:17,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:56:17,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:56:17,859 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:56:17,861 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:19,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:19,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:22,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:22,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:56:22,490 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:56:22,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:56:22,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:56:27,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:27,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:27,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:28,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:28,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:56:28,206 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:56:33,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:38,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:39,715 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:56:42,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:42,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:42,234 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:43,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:48,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:53,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:56:57,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:56:57,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:56:57,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:56:58,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:03,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:08,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:09,716 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:57:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:57:12,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:12,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:13,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:16,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,542 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_1b4ddf79dc604699f630.png with policy now +2023-07-12 08:57:16,551 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_259ec0830802f8053fe5.png with policy now +2023-07-12 08:57:16,561 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,561 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_bb1524320261cb0216db.png with policy now +2023-07-12 08:57:16,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:57:16,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2130_0099dae297478bd3ce67.png with policy now +2023-07-12 08:57:16,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:16,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:16,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:16,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:16,875 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/50gnvfoy-media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 08:57:16,877 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4aszxl8e-media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 08:57:16,903 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nn7vup5m-media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 08:57:16,935 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3s03mdg-media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 08:57:18,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:22,480 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:22,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:22,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 08:57:22,481 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 08:57:22,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 08:57:22,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 08:57:22,483 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:57:22,837 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pax3wzuv-media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 08:57:22,854 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0aa9eveu-media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 08:57:22,878 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uwy7803t-media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 08:57:22,925 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/72e3ul1v-media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 08:57:24,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:26,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:28,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:57:28,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:28,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:29,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:31,496 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:34,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:35,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:35,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:35,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:35,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:39,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:57:40,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:40,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:40,512 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:43,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:43,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:57:43,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:45,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:45,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:45,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:45,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:46,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:49,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:49,528 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:57:52,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:55,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:57:55,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:57:55,044 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:57:55,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:57:57,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:57:58,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:57:58,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:57:58,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:57:58,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:02,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:06,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:06,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:06,021 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:06,023 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:07,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:07,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:08,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:09,720 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:58:13,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:13,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:13,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:13,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:58:16,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:16,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:16,836 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:16,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:16,838 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:18,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:24,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:25,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:25,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:27,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:27,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:27,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:27,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:29,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:29,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:29,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:29,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:58:34,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:34,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:34,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:37,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:37,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:37,808 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:37,810 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:39,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:58:39,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:43,805 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:43,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:44,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:44,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:44,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:58:45,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:47,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:47,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:47,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:47,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:50,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:52,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:58:52,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:58:52,779 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:58:52,780 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:58:52,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:58:52,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:58:56,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:58:59,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:58:59,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:58:59,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:01,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:01,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:59:06,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:09,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:59:11,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:14,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:14,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:14,519 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:17,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:22,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:27,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:29,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:29,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:29,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:32,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:37,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:39,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 08:59:43,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:44,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:44,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:44,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 08:59:47,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_825c97191445f88d10ad.png with policy now +2023-07-12 08:59:47,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,068 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_ebe038f09b0a2e5c2779.png with policy now +2023-07-12 08:59:47,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_69c0860f204684fe6617.png with policy now +2023-07-12 08:59:47,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 08:59:47,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 08:59:47,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 08:59:47,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2140_a9fe0e5593aba42cc7e6.png with policy now +2023-07-12 08:59:47,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:59:47,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:59:47,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 08:59:47,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 08:59:47,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 08:59:47,431 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/znw8p4f3-media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 08:59:47,442 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fit8o2qw-media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 08:59:47,480 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ll2wqavw-media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 08:59:47,493 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ls1ppj1s-media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 08:59:47,880 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2qc1coky-media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 08:59:47,893 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqvkn1pr-media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 08:59:49,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:54,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:56,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 08:59:56,145 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 08:59:56,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 08:59:56,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 08:59:56,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 08:59:56,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 08:59:56,681 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bjeftafh-media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 08:59:56,713 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/prbunew4-media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 08:59:59,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 08:59:59,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 08:59:59,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 08:59:59,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:04,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:05,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:06,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:06,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:06,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:06,054 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:09,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:00:10,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:14,174 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:14,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:15,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:00:15,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:00:15,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:15,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:15,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:15,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:15,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:16,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:22,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:23,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:23,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:26,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:26,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:26,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:26,005 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:28,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:30,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:00:30,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:00:30,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:32,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:32,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:33,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:35,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:35,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:35,888 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:35,890 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:39,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:39,725 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:00:41,226 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:41,227 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:44,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:45,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:00:45,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:00:45,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:00:46,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:46,286 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:46,286 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:46,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:49,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:50,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:50,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:00:55,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:00:56,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:00:56,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:00:56,226 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:00:56,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:00:59,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:00:59,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:00,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:00,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:00,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:00,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:06,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:01:06,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:06,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:01:06,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:01:06,058 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:01:08,442 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:01:08,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:09,726 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:01:11,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:16,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:16,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:16,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:16,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:01:16,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:01:16,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:16,983 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:01:16,986 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:01:17,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:01:17,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:22,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:01:22,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:01:22,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:01:22,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:22,138 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:01:26,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:01:26,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:01:27,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:31,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:33,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:38,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:39,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:01:43,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:46,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:01:46,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:01:46,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:01:49,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:54,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:01:59,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:01,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:01,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:01,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:05,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:09,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:02:10,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:15,730 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:16,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:16,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:16,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:16,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_46dad1ead66f697f0698.png with policy now +2023-07-12 09:02:16,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,793 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_f451e47ddafe0bf1fcc6.png with policy now +2023-07-12 09:02:16,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_f44d5fe284ffc3b74829.png with policy now +2023-07-12 09:02:16,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:02:16,812 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2150_4f03c9db92527c876ba9.png with policy now +2023-07-12 09:02:16,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:16,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:16,852 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:16,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:17,124 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sb5l3umv-media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 09:02:17,126 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/io2ihe5k-media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 09:02:17,136 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2utpcrkj-media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 09:02:17,139 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q9jbw4kn-media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 09:02:20,740 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:20,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 09:02:20,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 09:02:20,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 09:02:20,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 09:02:20,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:02:21,071 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mm3dyh0f-media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 09:02:21,090 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m2xznws5-media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 09:02:21,128 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f9dq57u7-media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 09:02:21,148 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bejc8ccj-media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 09:02:21,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:26,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:27,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:29,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:31,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:31,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:31,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:36,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:36,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:36,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:36,437 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:37,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:38,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:38,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:39,729 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:02:42,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:46,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:02:46,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:02:46,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:02:46,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:46,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:46,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:46,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:47,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:47,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:47,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:53,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:02:56,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:02:56,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:02:56,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:02:56,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:02:56,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:02:56,803 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:02:59,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:01,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:01,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:01,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:04,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:05,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:06,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:06,839 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:06,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:06,841 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:09,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:09,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:03:14,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:14,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:14,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:16,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:16,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:16,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:17,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:17,354 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:17,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:17,359 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:19,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:24,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:24,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:25,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:27,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:27,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:27,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:27,352 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:30,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:31,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:31,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:31,786 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:33,044 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:33,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:36,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:37,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:37,522 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:37,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:37,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:39,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:03:41,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:42,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:42,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:47,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:03:47,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:03:47,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:03:47,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:47,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:47,836 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:47,836 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:47,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:51,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:03:51,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:03:52,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:03:53,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:03:53,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:03:53,056 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:03:53,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:03:58,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:00,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:04:00,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:04:02,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:02,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:02,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:04,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:09,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:09,733 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:04:14,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:17,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:17,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:17,289 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:20,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:25,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:30,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:32,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:32,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:32,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:35,518 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:39,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:04:40,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:45,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:47,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_f78f21bafff0abaf0089.png with policy now +2023-07-12 09:04:47,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_1ee652c4e539d1e07cce.png with policy now +2023-07-12 09:04:47,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,198 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_b97ae163d55862639f70.png with policy now +2023-07-12 09:04:47,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:04:47,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2160_ec40a3df133e538d57f3.png with policy now +2023-07-12 09:04:47,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:04:47,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:04:47,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:04:47,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:04:47,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:04:47,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:04:47,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:04:47,497 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9411z14-media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 09:04:47,516 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/01hwe2qk-media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 09:04:47,523 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3rlhwyqe-media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 09:04:47,603 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7u6x1jrq-media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 09:04:50,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:54,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:04:54,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:04:54,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 09:04:54,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 09:04:54,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 09:04:54,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 09:04:54,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:04:54,714 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y1wi1297-media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 09:04:54,775 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bcj4s9uc-media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 09:04:54,790 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7lldtcou-media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 09:04:54,794 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgcvtx6r-media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 09:04:56,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:04:58,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:01,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:02,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:05:02,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:02,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:03,388 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:07,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:07,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:07,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:07,509 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:07,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:09,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:05:12,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:12,404 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:12,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:17,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:05:17,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:17,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:17,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:17,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:17,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:17,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:17,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:21,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:21,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:23,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:27,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:27,451 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:27,451 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:27,453 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:29,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:30,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:30,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:32,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:32,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:34,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:37,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:37,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:37,604 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:37,606 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:39,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:39,451 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:39,736 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:05:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:46,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:48,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:05:48,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:05:48,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:05:48,466 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:48,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:48,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:48,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:48,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:05:52,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:57,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:05:57,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:05:57,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:05:58,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:05:58,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:05:58,428 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:05:58,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:02,716 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:03,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:03,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:03,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:06,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:06:06,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:08,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:08,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:06:08,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:06:08,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:06:08,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:09,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:06:13,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:15,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:06:15,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:18,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:18,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:18,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:18,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:06:18,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:18,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:06:18,739 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:06:18,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:23,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:06:23,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:06:23,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:06:23,497 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:06:24,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:24,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:06:24,726 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:29,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:33,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:33,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:33,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:33,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:06:34,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:39,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:39,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:06:44,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:48,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:06:48,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:06:48,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:06:50,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:06:55,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:00,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:03,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:03,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:03,426 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:05,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:09,741 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:07:10,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:15,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:17,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,654 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_36b4d3a784d14fee51a0.png with policy now +2023-07-12 09:07:17,663 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_33d957838c074e90f6a2.png with policy now +2023-07-12 09:07:17,674 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_533582fdbffc2ddc97ac.png with policy now +2023-07-12 09:07:17,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:07:17,684 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2170_def8ab43612eed95556d.png with policy now +2023-07-12 09:07:17,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:17,727 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:17,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:17,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:17,979 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q6v4fsdl-media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 09:07:17,986 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wajsl1k1-media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 09:07:17,994 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rvke0zc0-media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 09:07:18,008 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6b9s5mru-media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 09:07:18,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:18,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:18,427 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:19,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:07:19,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 09:07:19,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 09:07:19,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 09:07:19,017 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 09:07:19,018 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:07:19,389 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x1jdorgt-media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 09:07:19,407 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jx8096u7-media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 09:07:19,449 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i6p9aolc-media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 09:07:19,452 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3mv5yx9m-media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 09:07:21,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:26,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:27,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:28,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:32,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:33,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:33,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:33,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:36,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:36,334 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:36,334 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:36,336 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:37,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:07:37,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:38,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:39,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:07:43,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:46,146 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:46,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:46,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:46,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:46,239 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:48,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:07:48,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:07:48,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:07:48,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:54,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:07:55,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:07:55,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:07:56,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:07:56,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:07:56,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:07:56,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:07:59,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:03,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:03,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:03,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:04,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:04,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:05,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:06,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:06,335 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:06,336 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:06,338 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:09,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:08:10,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:13,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:13,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:15,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:16,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:16,348 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:16,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:16,350 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:18,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:18,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:18,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:21,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:22,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:22,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:26,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:26,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:26,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:26,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:26,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:31,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:31,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:32,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:33,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:33,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:33,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:36,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:36,349 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:36,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:36,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:37,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:39,744 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:08:40,428 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:40,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:42,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:46,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:46,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:46,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:46,902 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:47,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:49,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:08:49,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:08:49,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:08:49,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:49,444 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:51,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:08:51,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:08:51,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:08:51,914 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:08:53,590 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:08:58,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:08:58,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:08:58,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:03,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:04,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:04,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:04,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:09,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:09,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:09:14,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:19,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:19,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:19,316 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:25,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:30,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:34,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:34,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:34,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:35,481 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:39,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:09:40,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:45,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:45,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_d63b39c78c18b127efec.png with policy now +2023-07-12 09:09:45,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,781 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_f1a0da8e451df830c88e.png with policy now +2023-07-12 09:09:45,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_5d7322114061dc65484d.png with policy now +2023-07-12 09:09:45,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:09:45,802 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2180_f5a9c542fcce2ab79bf5.png with policy now +2023-07-12 09:09:45,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:09:45,843 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:09:45,844 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:09:45,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:09:46,105 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rw4c7c7y-media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 09:09:46,108 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q4rhba8p-media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 09:09:46,128 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6k36utwc-media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 09:09:46,133 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9c01fehe-media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 09:09:49,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:09:49,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:09:49,429 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:09:50,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 09:09:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 09:09:52,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 09:09:52,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 09:09:52,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:09:53,099 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eqv3xulz-media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 09:09:53,107 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3o8z8ld2-media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 09:09:53,145 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ze9hii2-media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 09:09:53,153 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rkbuiiee-media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 09:09:55,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:09:56,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:01,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:01,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:04,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:04,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:04,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:05,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:05,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:05,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:05,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:07,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:09,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:10:10,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:10,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:12,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:14,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:14,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:14,817 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:14,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:18,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:19,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:19,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:19,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:19,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:19,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:23,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:25,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:25,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:25,148 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:25,149 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:28,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:28,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:28,804 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:34,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:35,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:35,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:35,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:35,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:35,578 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:35,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:35,580 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:37,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:37,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:39,749 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:10:39,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:45,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:46,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:46,271 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:46,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:46,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:10:46,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:10:46,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:50,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:10:50,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:10:50,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:10:50,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:56,042 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:10:56,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:10:56,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:10:56,352 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:10:56,353 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:10:56,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:01,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:05,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:11:05,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:05,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:11:05,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:05,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:06,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:11:06,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:11:06,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:11:06,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:07,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:09,750 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:11:12,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:14,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:11:14,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:17,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:11:17,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:11:17,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:11:17,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:18,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:20,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:11:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:20,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:22,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:11:22,552 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:11:22,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:11:22,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:11:23,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:11:23,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:24,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:29,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:32,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:11:34,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:35,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:35,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:39,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:39,751 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:11:44,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:49,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:11:50,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:11:50,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:11:50,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:11:55,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:00,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:05,511 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:05,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:09,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:12:10,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:15,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:16,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_b89f7b812f07101e7bc5.png with policy now +2023-07-12 09:12:16,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_51f75fba279dc7d2d626.png with policy now +2023-07-12 09:12:16,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,930 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_cd64ca37330df964f268.png with policy now +2023-07-12 09:12:16,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:12:16,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2190_cd935f0213029522bb6c.png with policy now +2023-07-12 09:12:16,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:16,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:16,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:16,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:12:17,234 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lpfwbjv1-media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 09:12:17,260 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j2luf954-media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 09:12:17,293 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1duao47h-media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 09:12:17,308 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6egr8yjc-media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 09:12:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:12:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 09:12:17,390 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 09:12:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 09:12:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 09:12:17,391 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:12:17,777 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ot3oq1ws-media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 09:12:17,781 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nsc28c29-media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 09:12:17,792 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v9jqywm8-media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 09:12:17,832 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h86z7twp-media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 09:12:20,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:20,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:20,511 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:21,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:26,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:26,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:27,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:32,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:35,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:35,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:35,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:35,657 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:36,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:36,224 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:36,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:36,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:12:38,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:39,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:12:44,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:44,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:12:44,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:45,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:45,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:45,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:45,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:12:49,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:50,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:12:50,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:12:50,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:12:53,463 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:12:53,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:12:54,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:12:56,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:12:56,378 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:12:56,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:12:56,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:00,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:02,478 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:02,479 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:06,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:06,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:06,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:06,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:08,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:08,470 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:08,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:08,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:09,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:13:11,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:11,494 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:13,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:16,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:16,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:16,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:16,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:18,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:20,694 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:20,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:21,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:21,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:21,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:24,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:27,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:27,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:27,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:27,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:29,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:29,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:30,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:35,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:36,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:36,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:36,694 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:37,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:37,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:37,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:37,335 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:38,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:38,727 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:39,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:13:40,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:46,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:47,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:48,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:48,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:48,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:48,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:51,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:13:52,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:13:52,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:13:52,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:13:53,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:13:53,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:13:53,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:13:53,211 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:13:56,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:13:56,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:13:57,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:02,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:07,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:07,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:07,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:08,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:09,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:14:13,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:18,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:22,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:22,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:22,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:24,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:29,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:34,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:37,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:37,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:37,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:39,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:14:39,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:44,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:47,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_963ea35b0f8a3f28bf7f.png with policy now +2023-07-12 09:14:47,261 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,261 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_5479be16d55799087c1d.png with policy now +2023-07-12 09:14:47,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,272 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_485475519d4a6c767788.png with policy now +2023-07-12 09:14:47,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:14:47,301 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2200_8e5e62fe39fe252d734c.png with policy now +2023-07-12 09:14:47,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:14:47,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:14:47,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:14:47,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:14:47,566 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1l2s4df6-media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 09:14:47,574 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wg3crt76-media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 09:14:47,596 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uj2ipm5r-media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 09:14:47,632 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/plshgnnc-media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 09:14:50,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:51,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:14:51,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:14:51,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 09:14:51,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 09:14:51,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 09:14:51,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 09:14:51,035 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:14:51,366 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t6pxjueg-media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 09:14:51,378 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/90nk4zxp-media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 09:14:51,409 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ohfo08na-media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 09:14:51,426 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/51mg9qmg-media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 09:14:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:14:52,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:14:52,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:14:55,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:14:57,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:00,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:01,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:06,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:06,122 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:06,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:06,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:07,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:08,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:15:08,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:08,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:09,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:09,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:09,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:15:12,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:15,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:15,896 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:15,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:15,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:17,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:18,080 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:18,081 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:23,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:15:23,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:23,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:23,168 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:25,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:25,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:25,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:25,773 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:27,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:27,096 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:28,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:33,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:36,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:36,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:36,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:36,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:36,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:36,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:38,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:38,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:39,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:39,761 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:15:44,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:45,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:45,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:45,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:45,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:45,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:15:50,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:53,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:15:53,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:15:54,371 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:15:54,372 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:15:55,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:15:55,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:15:55,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:15:55,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:15:55,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:01,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:03,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:03,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:06,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:16:06,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:06,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:16:06,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:16:06,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:08,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:08,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:08,402 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:09,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:16:11,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:12,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:12,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:17,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:17,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:16:17,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:16:17,498 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:16:17,500 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:21,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:21,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:21,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:16:21,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:16:21,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:16:21,903 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:16:22,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:23,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:23,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:23,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:27,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:30,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:16:30,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:16:32,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:37,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:38,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:38,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:38,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:39,764 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:16:42,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:47,766 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:52,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:16:53,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:16:53,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:16:53,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:16:58,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:03,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:08,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:08,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:08,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:08,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:09,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:17:13,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:16,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,110 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_3d504d65d8985e43c4df.png with policy now +2023-07-12 09:17:16,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_7e6627623494cc0c7f15.png with policy now +2023-07-12 09:17:16,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,130 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_14e783bea121b92109bd.png with policy now +2023-07-12 09:17:16,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:17:16,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2210_b4ca94832829e41b0eaa.png with policy now +2023-07-12 09:17:16,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:16,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:16,183 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:16,185 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:16,460 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zdly73nn-media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 09:17:16,464 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m2i320el-media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 09:17:16,512 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jsjue2x9-media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 09:17:16,572 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c3nfg7yw-media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 09:17:19,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:23,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:23,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:23,692 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:24,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:17:24,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:24,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 09:17:24,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 09:17:24,715 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 09:17:24,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 09:17:24,716 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:17:25,070 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/42q8qagz-media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 09:17:25,075 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/juojg8r7-media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 09:17:25,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:25,089 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8jm7ewna-media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 09:17:25,115 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e3qnwl3s-media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 09:17:26,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:30,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:33,731 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:35,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:35,264 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:35,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:35,266 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:36,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:39,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:39,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:39,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:39,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:17:41,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:42,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:17:42,747 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:45,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:45,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:45,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:45,503 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:47,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:51,763 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:17:51,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:17:53,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:17:54,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:17:54,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:17:54,608 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:17:55,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:17:55,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:17:55,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:17:55,175 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:17:58,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:00,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:00,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:04,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:05,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:05,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:05,820 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:05,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:09,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:18:09,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:09,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:09,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:09,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:09,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:10,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:15,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:15,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:15,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:15,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:15,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:18,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:18,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:20,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:25,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:25,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:25,206 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:25,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:25,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:25,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:25,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:25,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:28,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:28,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:31,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:35,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:35,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:35,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:35,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:37,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:37,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:37,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:39,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:18:40,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:40,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:40,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:42,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:45,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:45,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:45,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:45,526 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:46,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:46,046 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:48,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:50,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:18:50,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:18:50,672 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:18:50,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:18:54,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:18:55,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:18:55,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:18:55,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:18:55,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:18:55,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:18:59,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:04,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:09,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:09,770 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:19:10,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:10,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:10,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:14,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:19,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:24,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:25,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:25,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:25,515 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:29,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:34,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:39,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:39,771 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:19:40,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:40,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:40,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:44,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:45,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,183 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_fd2b6676675f447dd0bb.png with policy now +2023-07-12 09:19:45,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_a2b2236edc7a209199e4.png with policy now +2023-07-12 09:19:45,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,205 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_be095f7407792a1d8c2b.png with policy now +2023-07-12 09:19:45,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:19:45,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2220_7995e8c15e47416a5a98.png with policy now +2023-07-12 09:19:45,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:19:45,257 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:19:45,257 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:19:45,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:19:45,528 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/91yrh4m9-media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 09:19:45,542 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3022emnu-media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 09:19:45,560 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g9fmuyjy-media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 09:19:45,572 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2t89gahv-media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 09:19:49,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:19:49,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:19:49,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 09:19:49,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 09:19:49,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 09:19:49,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 09:19:49,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:19:49,686 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/whp60vb0-media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 09:19:49,709 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gz71cxt5-media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 09:19:49,732 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0r8214o1-media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 09:19:49,754 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k6n4kazr-media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 09:19:50,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:54,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:19:55,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:19:55,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:19:55,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:19:55,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:19:58,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:00,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:04,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:04,256 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:04,256 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:04,259 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:06,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:07,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:07,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:09,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:20:10,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:10,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:10,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:11,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:14,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:14,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:14,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:14,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:16,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:16,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:17,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:23,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:24,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:24,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:24,972 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:24,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:25,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:25,411 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:25,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:25,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:25,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:28,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:33,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:34,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:35,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:35,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:35,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:35,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:38,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:39,774 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:20:40,722 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:40,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:40,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:43,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:43,443 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:44,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:45,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:45,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:45,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:45,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:20:50,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:52,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:20:52,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:20:55,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:20:55,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:20:55,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:20:55,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:20:55,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:20:55,813 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:20:55,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:20:55,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:01,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:01,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:01,666 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:05,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:21:05,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:21:05,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:21:05,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:06,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:09,775 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:21:10,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:10,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:10,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:10,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:10,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:12,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:17,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:21:17,336 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:21:17,337 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:21:17,338 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:18,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:19,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:19,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:21,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:21:21,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:21:21,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:21:21,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:21:23,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:26,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:26,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:26,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:28,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:21:28,713 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:21:29,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:34,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:39,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:39,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:21:41,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:41,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:41,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:44,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:49,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:54,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:21:56,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:21:56,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:21:56,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:21:59,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:04,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:09,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:09,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:22:11,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:11,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:11,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:14,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:16,162 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_fa6aae454ea563a1568c.png with policy now +2023-07-12 09:22:16,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_bba8aa2011cbb2b9e998.png with policy now +2023-07-12 09:22:16,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_cc1f25e32b609b2e7f13.png with policy now +2023-07-12 09:22:16,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:22:16,191 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2230_c694e9556fc95a54794c.png with policy now +2023-07-12 09:22:16,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:16,232 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:16,232 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:16,234 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:16,501 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/901tao6y-media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 09:22:16,534 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ltr8cau-media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 09:22:16,550 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lz0bh5p4-media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 09:22:16,567 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o173z6gt-media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 09:22:19,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:22,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:22,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:22,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 09:22:22,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 09:22:22,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 09:22:22,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 09:22:22,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:22:23,336 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aptgboqj-media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 09:22:23,341 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g4b3swqe-media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 09:22:23,353 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9n1uhzmk-media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 09:22:23,355 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/khzbviwm-media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 09:22:25,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:26,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:26,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:26,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:26,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:30,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:32,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:36,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:36,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:36,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:36,022 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:36,023 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:39,779 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:22:41,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:41,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:41,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:41,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:41,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:42,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:45,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:45,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:45,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:45,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:47,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:50,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:50,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:22:52,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:55,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:22:55,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:22:55,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:22:55,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:22:56,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:22:56,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:22:56,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:22:58,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:22:59,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:22:59,063 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:03,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:05,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:05,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:05,588 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:05,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:08,077 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:09,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:09,780 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:23:11,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:11,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:11,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:15,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:16,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:16,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:16,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:16,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:17,093 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:17,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:20,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:26,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:26,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:26,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:26,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:26,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:26,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:26,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:26,517 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:31,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:35,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:35,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:36,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:36,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:36,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:36,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:37,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:39,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:23:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:41,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:41,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:42,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:44,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:44,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:47,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:47,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:47,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:47,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:48,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:52,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:23:52,269 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:23:52,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:23:52,271 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:23:53,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:23:53,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:23:53,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:23:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:23:56,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:23:56,518 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:23:59,554 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:04,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:09,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:09,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:24:11,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:11,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:11,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:14,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:19,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:24,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:26,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:26,594 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:26,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:30,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:35,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:39,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:24:40,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:41,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:41,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:41,737 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:45,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:46,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_c76d157bbb397161c214.png with policy now +2023-07-12 09:24:46,743 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,743 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_cbb566c39078908ba8b3.png with policy now +2023-07-12 09:24:46,753 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,753 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_17e81b81f9d5ecefe6ad.png with policy now +2023-07-12 09:24:46,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:24:46,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2240_77e784d7b8aacf9759c6.png with policy now +2023-07-12 09:24:46,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:24:46,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:24:46,807 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:24:46,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:24:47,047 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gu0dv404-media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 09:24:47,057 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yg21fw7i-media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 09:24:47,064 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9b0wvifq-media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 09:24:47,110 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1kas1vxv-media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 09:24:47,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:24:47,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 09:24:47,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 09:24:47,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 09:24:47,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 09:24:47,631 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:24:48,001 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fya6adsq-media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 09:24:48,002 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhdcjpxn-media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 09:24:48,009 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tvvv6ft1-media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 09:24:48,011 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eu7b8vnz-media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 09:24:50,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:56,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:24:56,646 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:24:56,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:24:56,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:24:56,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:24:56,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:02,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:05,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:05,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:05,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:05,848 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:06,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:06,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:07,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:09,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:25:12,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:12,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:12,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:12,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:15,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:15,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:15,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:15,180 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:16,674 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:16,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:18,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:24,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:26,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:26,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:26,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:26,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:26,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:26,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:27,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:27,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:27,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:29,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:34,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:36,707 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:37,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:37,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:37,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:37,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:39,786 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:25:39,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:42,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:42,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:42,381 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:45,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:46,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:46,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:47,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:47,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:47,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:47,920 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:25:50,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:56,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:25:56,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:25:56,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:25:58,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:25:58,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:25:58,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:25:58,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:25:58,646 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:25:58,646 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:25:58,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:01,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:06,953 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:26:06,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:26:07,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:09,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:26:09,597 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:26:09,597 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:26:09,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:09,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:26:12,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:13,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:13,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:26:13,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:16,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:26:16,970 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:26:17,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:19,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:26:19,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:26:19,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:26:19,387 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:22,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:24,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:26:24,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:26:24,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:26:24,041 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:26:26,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:26:26,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:26:28,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:28,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:28,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:33,153 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:38,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:39,788 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:26:43,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:26:43,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:43,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:43,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:48,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:53,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:26:58,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:26:58,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:26:58,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:26:59,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:04,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:09,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:09,790 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:27:13,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:27:13,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:13,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:14,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:18,723 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_ddcd1a1e8b54f46f7b26.png with policy now +2023-07-12 09:27:18,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,732 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_c1f37bde1c536851ff31.png with policy now +2023-07-12 09:27:18,742 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png with policy now +2023-07-12 09:27:18,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:27:18,752 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2250_84e331f8cb88258072e5.png with policy now +2023-07-12 09:27:18,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:18,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:18,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:18,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:27:19,095 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vpghxriw-media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 09:27:19,098 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1rkjmuhw-media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 09:27:19,113 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ueey357p-media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 09:27:19,129 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cq034kyn-media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 09:27:19,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:25,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:27,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:27:27,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:27,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 09:27:27,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 09:27:27,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 09:27:27,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 09:27:27,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:27:27,676 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qr1w4g10-media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 09:27:27,690 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f4975hn7-media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 09:27:27,694 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fm4miood-media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 09:27:27,701 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c0f2lysp-media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 09:27:28,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:28,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:27:28,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:29,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:31,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:37,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:37,291 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:38,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:38,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:38,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:38,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:27:39,791 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:27:42,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:43,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:27:43,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:43,401 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:47,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:47,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:27:47,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:48,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:48,468 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:48,468 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:48,470 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:27:53,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:57,324 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:27:57,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:27:58,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:27:58,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:27:58,400 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:27:58,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:27:58,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:27:58,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:27:58,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:03,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:07,342 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:07,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:08,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:09,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:09,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:09,362 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:09,364 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:09,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:28:13,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:28:13,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:28:13,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:28:13,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:17,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:17,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:18,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:18,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:18,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:18,985 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:18,987 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:24,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:27,570 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:27,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:28,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:28:28,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:28:28,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:28:29,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:29,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:29,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:29,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:30,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:35,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:37,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:37,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:39,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:28:40,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:40,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:40,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:40,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:41,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:45,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:28:45,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:28:45,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:28:46,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:47,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:47,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:28:51,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:51,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:51,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:51,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:52,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:55,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:28:55,762 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:28:55,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:28:55,764 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:28:57,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:28:57,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:28:57,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:29:00,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:00,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:00,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:03,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:07,634 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:29:08,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:09,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:29:13,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:15,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:15,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:15,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:19,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:24,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:29,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:30,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:30,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:30,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:34,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:39,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:39,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:29:44,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:29:45,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:29:45,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:29:50,505 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,505 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_1d09dab1e10018c0f5a2.png with policy now +2023-07-12 09:29:50,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:50,521 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,521 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_f5085a2630d7643ccb37.png with policy now +2023-07-12 09:29:50,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,531 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_ea013c7b85e22fd83c96.png with policy now +2023-07-12 09:29:50,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:29:50,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2260_f2fc182f190a6e4d3266.png with policy now +2023-07-12 09:29:50,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:29:50,584 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:29:50,585 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:29:50,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:29:50,923 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cmh0wjhi-media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 09:29:50,929 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/69ue3b1y-media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 09:29:50,936 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ks1bxxz-media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 09:29:50,959 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h4c9vzbe-media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 09:29:56,363 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:29:57,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:29:57,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:29:57,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 09:29:57,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 09:29:57,914 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 09:29:57,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 09:29:57,915 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:29:58,378 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hegycz9f-media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 09:29:58,383 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nb54b5zf-media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 09:29:58,387 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f4kd75cm-media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 09:29:58,388 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eutq3b0l-media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 09:30:00,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:00,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:00,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:00,465 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:01,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:07,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:07,929 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:09,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:09,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:09,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:09,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:09,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:30:12,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:15,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:15,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:15,741 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:17,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:17,946 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:18,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:18,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:18,983 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:18,983 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:18,985 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:23,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:27,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:27,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:29,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:29,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:29,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:29,857 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:29,859 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:32,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:32,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:32,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:35,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:37,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:37,978 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:39,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:30:39,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:39,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:39,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:39,852 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:40,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:45,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:47,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:30:47,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:30:47,680 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:30:47,994 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:47,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:30:50,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:30:50,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:30:50,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:30:50,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:30:51,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:56,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:30:58,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:30:58,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:00,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:00,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:00,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:00,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:01,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:02,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:02,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:02,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:07,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:08,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:31:08,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:09,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:31:11,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:11,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:11,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:11,567 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:12,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:17,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:17,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:17,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:17,909 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:18,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:31:18,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:22,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:22,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:22,620 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:22,622 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:23,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:27,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:31:27,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:31:27,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:31:27,153 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:31:28,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:31:28,257 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:28,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:33,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:34,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:38,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:31:39,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:39,800 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:31:44,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:48,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:31:48,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:31:48,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:31:50,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:31:55,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:00,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:03,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:03,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:03,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:06,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:09,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:32:11,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:16,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:18,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:18,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:18,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:21,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,737 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_57fbdf8438f8224664e4.png with policy now +2023-07-12 09:32:21,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,746 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_66b7fc2a071b6e92e172.png with policy now +2023-07-12 09:32:21,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,756 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_1db6dc00cd88a44aa4f5.png with policy now +2023-07-12 09:32:21,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:32:21,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2270_506c3fdf835cc8b9e9bc.png with policy now +2023-07-12 09:32:21,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:21,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:32:21,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:32:21,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:21,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:32:22,041 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1b6g8ee9-media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 09:32:22,054 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dmdn8byr-media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 09:32:22,076 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oxhlr45e-media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 09:32:22,081 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n97yr37c-media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 09:32:27,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:32:28,540 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:32:28,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 09:32:28,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 09:32:28,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 09:32:28,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 09:32:28,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:32:28,887 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9724g1r-media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 09:32:28,900 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wdi0ytjs-media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 09:32:28,924 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ekg5or7-media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 09:32:28,962 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yia8inx3-media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 09:32:31,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:33,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:33,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:33,039 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:33,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:38,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:38,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:32:39,803 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:32:40,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:40,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:32:40,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:32:40,219 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:32:43,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:48,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:32:48,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:32:48,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:32:48,571 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:32:48,572 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:32:49,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:49,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:32:49,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:32:49,988 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:32:49,990 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:32:54,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:32:58,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:32:58,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:00,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:00,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:00,609 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:00,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:00,611 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:03,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:03,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:03,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:05,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:08,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:08,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:09,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:33:11,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:12,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:12,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:12,019 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:12,021 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:16,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:18,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:18,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:19,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:19,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:19,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:21,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:21,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:21,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:21,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:22,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:28,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:28,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:28,840 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:32,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:32,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:32,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:32,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:33,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:34,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:34,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:34,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:38,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:38,855 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:39,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:39,805 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:33:43,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:43,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:43,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:43,570 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:45,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:48,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:48,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:33:50,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:33:50,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:33:50,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:33:50,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:53,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:53,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:53,890 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:53,892 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:55,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:33:58,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:33:58,760 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:33:58,760 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:33:58,762 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:33:58,889 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:33:58,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:34:01,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:05,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:05,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:05,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:06,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:08,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:34:09,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:34:12,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:17,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:20,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:20,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:20,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:22,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:27,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:32,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:36,129 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:38,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:39,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:34:43,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:48,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:51,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:34:51,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:34:51,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:34:53,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,135 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_d6d98a9fc2b412d909b2.png with policy now +2023-07-12 09:34:53,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,145 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_9e0da1558268d346b2ff.png with policy now +2023-07-12 09:34:53,154 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,154 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_a95379a0e1bb4ee5eefd.png with policy now +2023-07-12 09:34:53,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:34:53,165 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2280_50d6f6aebffcec48f478.png with policy now +2023-07-12 09:34:53,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:34:53,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:34:53,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:34:53,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:34:53,460 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1i9uvnua-media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 09:34:53,473 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/esu3txv3-media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 09:34:53,483 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a6n2e4v5-media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 09:34:53,494 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ky2zepcz-media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 09:34:54,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:34:59,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:34:59,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:34:59,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 09:34:59,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 09:34:59,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 09:34:59,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 09:34:59,190 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:34:59,538 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3l0w83xe-media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 09:34:59,542 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/n2j4noa5-media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 09:34:59,556 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t9slmzl9-media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 09:34:59,557 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/700rdeiv-media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 09:34:59,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:03,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:05,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:06,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:06,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:06,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:09,205 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:09,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:35:11,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:12,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:12,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:12,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:12,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:16,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:19,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:19,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:22,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:22,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:22,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:22,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:22,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:22,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:22,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:22,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:27,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:29,240 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:29,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:32,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:33,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:33,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:33,009 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:33,011 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:37,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:37,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:37,496 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:37,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:39,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:39,255 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:39,810 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:35:42,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:43,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:43,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:43,290 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:43,292 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:47,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:49,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:49,273 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:35:52,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:35:52,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:35:52,637 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:35:53,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:35:53,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:53,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:35:53,200 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:35:53,202 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:35:58,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:35:59,498 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:35:59,499 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:03,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:03,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:03,320 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:03,323 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:04,324 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:07,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:07,769 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:07,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:09,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:09,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:36:09,514 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:09,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:36:13,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:13,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:13,572 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:13,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:14,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:19,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:36:19,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:19,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:23,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:23,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:23,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:23,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:23,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:23,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:23,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:24,858 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:29,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:36:29,004 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:36:29,005 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:36:29,006 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:36:29,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:36:29,545 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:30,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:35,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:38,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:38,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:38,283 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:39,559 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:36:39,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:36:40,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:45,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:50,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:36:53,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:36:53,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:36:53,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:36:56,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:01,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:06,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:08,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:08,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:09,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:37:11,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:16,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:21,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:23,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:23,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:23,344 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,345 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_dc4de63f9960a0a31886.png with policy now +2023-07-12 09:37:23,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_d8ebb18636c7304a48a5.png with policy now +2023-07-12 09:37:23,346 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,346 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_8ab18a5e3a00fe86d6ef.png with policy now +2023-07-12 09:37:23,411 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:37:23,411 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2290_f6c9e9af5b280aff421b.png with policy now +2023-07-12 09:37:23,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:23,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:37:23,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:37:23,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:37:23,686 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/akepndb8-media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 09:37:23,711 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/11cxsso8-media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 09:37:23,751 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y5axkk39-media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 09:37:23,770 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ig6384fb-media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 09:37:27,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:29,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:37:29,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:37:29,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 09:37:29,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 09:37:29,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 09:37:29,844 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 09:37:29,845 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:37:30,226 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9h8h2fhl-media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 09:37:30,229 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7yoqm9p5-media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 09:37:30,230 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bxgdcyvj-media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 09:37:30,327 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/teu5jzdq-media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 09:37:33,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:33,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:38,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:38,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:38,369 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:39,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:37:39,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:37:43,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:43,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:37:43,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:37:43,070 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:37:44,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:49,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:49,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:37:49,875 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:37:53,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:37:53,235 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:37:53,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:37:53,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:37:53,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:37:53,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:37:55,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:37:59,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:37:59,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:00,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:03,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:03,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:03,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:03,141 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:05,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:08,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:08,284 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:09,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:38:09,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:09,909 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:11,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:13,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:13,425 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:13,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:13,427 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:16,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:19,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:19,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:22,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:23,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:38:23,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:23,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:23,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:23,782 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:23,783 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:23,785 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:27,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:30,131 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:30,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:33,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:33,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:33,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:33,684 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:33,686 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:38,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:38:38,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:38,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:38,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:39,817 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:38:40,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:40,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:44,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:44,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:44,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:44,387 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:44,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:49,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:50,162 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:38:50,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:38:53,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:38:53,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:38:53,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:38:54,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:54,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:54,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:54,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:38:55,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:38:59,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:38:59,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:38:59,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:38:59,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:39:00,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:39:00,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:39:00,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:05,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:08,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:08,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:08,809 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:09,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:39:10,194 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:39:10,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:15,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:20,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:24,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:24,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:24,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:26,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:31,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:36,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:39,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:39,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:39,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:39,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:39:41,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:46,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:51,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:39:53,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_229851204683a32b894d.png with policy now +2023-07-12 09:39:53,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_b9647059d362b9b7c491.png with policy now +2023-07-12 09:39:53,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png with policy now +2023-07-12 09:39:53,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:39:53,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2300_abce74fb7ce87882e4ef.png with policy now +2023-07-12 09:39:53,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:39:53,199 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:39:53,199 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:39:53,204 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:39:53,446 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vc0r03p0-media/images/validation_2300_229851204683a32b894d.png +2023-07-12 09:39:53,493 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/06ysbj9c-media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 09:39:53,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r5bmxrmr-media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 09:39:53,500 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8wscchzy-media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 09:39:54,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:39:54,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:39:54,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:39:57,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:00,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:00,472 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:00,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 09:40:00,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 09:40:00,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_229851204683a32b894d.png +2023-07-12 09:40:00,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 09:40:00,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:40:00,827 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c8l7fjnt-media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 09:40:00,847 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qcyjhn4j-media/images/validation_2300_229851204683a32b894d.png +2023-07-12 09:40:00,869 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fv8ec9yb-media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 09:40:00,887 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yuk3p8ze-media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 09:40:02,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:03,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:09,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:09,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:09,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:09,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:09,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:40:10,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:11,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:11,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:11,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:11,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:14,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:20,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:20,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:20,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:20,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:20,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:20,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:20,999 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:24,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:24,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:24,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:25,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:30,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:30,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:31,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:31,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:31,327 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:31,327 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:31,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:37,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:39,226 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:39,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:39,227 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:39,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:40:40,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:40,539 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:41,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:41,479 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:41,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:41,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:42,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:47,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:50,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:40:50,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:40:52,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:40:52,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:40:52,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:40:52,741 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:40:52,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:40:54,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:40:54,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:40:54,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:40:58,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:00,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:00,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:02,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:02,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:02,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:02,697 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:03,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:08,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:09,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:41:10,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:10,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:10,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:13,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:13,293 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:13,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:13,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:14,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:19,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:20,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:20,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:24,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:24,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:24,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:24,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:25,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:25,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:25,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:25,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:28,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:41:28,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:41:28,952 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:41:28,954 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:41:30,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:30,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:41:30,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:35,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:39,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:41:40,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:40,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:40,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:40,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:41:41,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:46,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:51,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:41:55,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:41:55,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:41:55,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:41:57,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:02,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:07,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:09,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:42:10,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:10,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:10,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:42:12,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:17,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:22,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:23,555 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_0b62299b4392d7dab3b5.png with policy now +2023-07-12 09:42:23,565 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,565 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_8adc40877878e2c5992a.png with policy now +2023-07-12 09:42:23,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,576 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_b8447334c2fa99557dab.png with policy now +2023-07-12 09:42:23,585 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:42:23,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2310_0a4470d06e9e5634fc30.png with policy now +2023-07-12 09:42:23,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:23,627 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:42:23,627 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:42:23,630 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:42:23,907 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aclp9wbq-media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 09:42:23,918 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5lt9lsx8-media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 09:42:23,922 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rgy602es-media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 09:42:23,938 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/40zgd1vr-media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 09:42:25,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:25,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:25,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:42:27,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:31,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:42:31,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:42:31,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 09:42:31,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:42:31,473 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8sl8k8n3-media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 09:42:31,487 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ch6dx0s-media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 09:42:31,489 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/csljihcl-media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 09:42:31,559 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pckir8mc-media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 09:42:33,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:33,512 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:38,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:39,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:42:40,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:40,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:40,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:42:41,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:42:42,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:42,230 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:42:42,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:42:42,233 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:42:44,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:49,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:51,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:42:51,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:42:52,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:42:52,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:42:52,898 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:42:52,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:42:54,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:42:55,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:42:55,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:42:55,319 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:00,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:01,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:01,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:03,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:03,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:03,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:03,097 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:05,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:09,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:43:10,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:10,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:10,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:10,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:11,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:11,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:13,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:13,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:13,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:13,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:15,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:20,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:21,378 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:21,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:23,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:23,297 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:23,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:23,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:25,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:25,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:25,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:25,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:31,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:31,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:31,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:33,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:33,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:33,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:33,988 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:36,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:39,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:43:40,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:40,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:40,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:41,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:41,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:42,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:45,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:45,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:45,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:45,168 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:47,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:51,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:43:51,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:43:53,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:43:55,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:43:55,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:43:55,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:43:55,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:43:55,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:43:55,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:43:55,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:43:58,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:00,440 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:44:00,441 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:44:00,442 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:44:00,443 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:44:01,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:44:01,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:44:04,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:09,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:09,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:44:10,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:10,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:10,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:14,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:19,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:24,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:25,763 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:25,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:25,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:29,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:34,843 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:39,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:44:40,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:40,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:40,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:40,845 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:45,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:50,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:44:54,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,863 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_6e0bdeefa42f0abef07e.png with policy now +2023-07-12 09:44:54,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,872 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_cad2712e0579795eae61.png with policy now +2023-07-12 09:44:54,883 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_8982b4ca24e18e3a2e66.png with policy now +2023-07-12 09:44:54,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:44:54,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2320_d536893289030f727428.png with policy now +2023-07-12 09:44:54,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:44:54,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:44:54,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:44:54,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:44:55,167 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s7xlp9ty-media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 09:44:55,176 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/czq0v793-media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 09:44:55,198 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g4kogq33-media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 09:44:55,257 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/43tild5v-media/images/validation_2320_d536893289030f727428.png +2023-07-12 09:44:55,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:44:55,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:44:55,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:44:56,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:01,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:01,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:01,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 09:45:01,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 09:45:01,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_d536893289030f727428.png +2023-07-12 09:45:01,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 09:45:01,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:45:02,338 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sw9twdfs-media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 09:45:02,347 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hsawhjl4-media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 09:45:02,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:02,358 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/31u2wnm1-media/images/validation_2320_d536893289030f727428.png +2023-07-12 09:45:02,382 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q1w72r7f-media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 09:45:04,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:07,405 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:09,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:45:10,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:10,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:10,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:11,976 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:12,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:14,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:14,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:14,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:14,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:18,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:21,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:21,995 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:23,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:24,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:24,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:24,989 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:24,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:26,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:26,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:26,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:29,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:32,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:32,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:34,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:34,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:34,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:34,953 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:34,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:39,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:45:40,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:41,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:41,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:41,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:42,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:42,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:45,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:45,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:45,066 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:45,067 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:46,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:51,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:45:52,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:45:52,050 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:45:56,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:45:56,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:45:56,881 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:45:56,883 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:45:56,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:45:56,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:45:56,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:45:56,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:02,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:02,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:02,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:06,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:06,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:06,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:06,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:07,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:09,834 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:46:11,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:11,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:11,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:12,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:12,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:12,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:17,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:17,472 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:17,472 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:17,474 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:18,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:22,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:22,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:23,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:27,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:27,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:27,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:27,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:27,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:28,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:32,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:46:32,290 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:46:32,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:46:32,293 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:46:32,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:46:32,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:33,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:38,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:39,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:46:42,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:42,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:42,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:42,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:46:44,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:49,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:54,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:46:57,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:46:57,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:46:57,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:46:59,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:04,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:09,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:09,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:47:12,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:47:12,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:12,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:15,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:20,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:25,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:26,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_9d435e637b3005c16f38.png with policy now +2023-07-12 09:47:26,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_c9b9321adf648747a0cb.png with policy now +2023-07-12 09:47:26,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,956 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_86f3d365c9fc4e6ab5e8.png with policy now +2023-07-12 09:47:26,966 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:47:26,966 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2330_1afc2708fa415a3e0f40.png with policy now +2023-07-12 09:47:27,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:27,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:47:27,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:47:27,009 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:47:27,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:47:27,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:27,112 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:27,278 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5r2h5c07-media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 09:47:27,282 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mr2yrq4n-media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 09:47:27,335 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/806apnxq-media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 09:47:27,337 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kcrcqqjk-media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 09:47:30,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:32,609 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:47:32,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:47:32,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 09:47:32,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 09:47:32,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 09:47:32,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 09:47:32,612 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:47:32,961 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bx28zwbj-media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 09:47:32,963 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3aixjuw0-media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 09:47:32,966 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/msm9ebru-media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 09:47:32,973 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gzythlgy-media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 09:47:36,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:36,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:39,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:47:41,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:42,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:42,107 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:42,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:47:45,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:45,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:47:45,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:47:45,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:47:46,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:52,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:47:52,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:47:52,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:47:56,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:47:56,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:47:56,168 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:47:56,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:47:57,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:47:57,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:47:57,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:47:57,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:02,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:02,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:03,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:06,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:06,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:06,277 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:06,279 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:08,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:09,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:48:12,675 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:12,676 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:13,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:48:13,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:13,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:13,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:17,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:17,348 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:17,349 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:17,351 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:19,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:22,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:22,691 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:25,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:28,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:28,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:28,449 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:28,451 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:28,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:28,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:30,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:32,911 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:32,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:36,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:38,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:38,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:38,541 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:38,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:39,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:48:41,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:42,926 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:42,927 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:43,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:43,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:46,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:48,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:48,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:48,665 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:48,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:48:52,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:52,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:48:52,943 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:48:57,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:48:58,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:48:58,774 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:48:59,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:48:59,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:48:59,032 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:48:59,034 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:49:02,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:49:02,958 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:49:03,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:03,717 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:49:03,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:49:03,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:49:03,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:49:09,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:09,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:49:12,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:49:12,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:49:13,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:13,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:13,889 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:49:14,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:19,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:24,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:28,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:28,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:28,940 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:49:30,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:35,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:39,842 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:49:40,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:44,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:44,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:44,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:49:46,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:51,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:56,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:49:58,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_e1920e85b34fdfb6609a.png with policy now +2023-07-12 09:49:58,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,207 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_65d2e2217439693f9468.png with policy now +2023-07-12 09:49:58,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,217 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_cfd6e181a1bb997a54d6.png with policy now +2023-07-12 09:49:58,227 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:49:58,227 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2340_d7a930a7a71d94c1211f.png with policy now +2023-07-12 09:49:58,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:49:58,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:49:58,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:49:58,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:49:58,512 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vf7yltlz-media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 09:49:58,533 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7k2p6pbh-media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 09:49:58,545 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9yqsklc1-media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 09:49:58,546 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/72epqs35-media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 09:49:59,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:49:59,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:49:59,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:01,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:03,263 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 09:50:03,264 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 09:50:03,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 09:50:03,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:50:03,594 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aura8mrm-media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 09:50:03,608 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8hgcvbik-media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 09:50:03,613 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rjr4z1o3-media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 09:50:03,751 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/njlkvtj9-media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 09:50:06,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:07,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:09,843 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:50:12,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:13,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:14,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:50:14,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:50:14,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:16,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:16,598 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:16,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:16,600 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:18,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:23,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:23,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:23,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:27,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:27,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:27,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:27,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:29,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:29,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:50:29,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:50:29,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:33,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:33,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:34,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:37,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:37,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:37,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:37,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:39,844 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:50:39,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:43,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:43,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:44,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:50:44,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:50:44,876 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:50:44,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:48,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:48,008 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:48,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:48,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:50:50,011 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:53,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:50:53,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:50:55,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:50:58,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:50:58,491 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:50:58,491 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:50:58,493 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:00,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:00,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:51:00,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:00,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:03,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:03,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:05,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:09,078 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:09,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:09,081 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:09,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:09,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:51:10,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:13,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:13,600 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:16,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:51:16,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:16,182 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:16,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:19,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:19,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:19,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:19,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:21,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:23,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:23,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:27,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:30,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:30,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:30,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:30,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:31,181 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:31,181 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:32,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:33,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:33,671 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:35,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:51:35,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:51:35,083 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:51:35,085 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:51:37,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:39,846 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:51:42,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:43,687 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:51:43,688 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:51:46,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:51:46,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:51:46,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:51:48,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:53,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:51:58,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:01,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:01,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:01,339 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:04,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:09,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:09,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:52:14,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:16,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:16,407 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:16,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:20,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:25,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:29,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,570 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_576ed58f083c4925ff4f.png with policy now +2023-07-12 09:52:29,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_6b91f2effcbccf59aafb.png with policy now +2023-07-12 09:52:29,589 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_7116b69ba382286251a6.png with policy now +2023-07-12 09:52:29,599 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:52:29,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2350_4776174cbccf4dcad7cf.png with policy now +2023-07-12 09:52:29,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:29,642 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:52:29,642 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:52:29,644 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:52:29,873 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/skd1n6u1-media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 09:52:29,892 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xmow84zg-media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 09:52:29,902 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dr9q5jht-media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 09:52:29,920 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y4dktnl2-media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 09:52:30,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:32,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:32,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:32,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:33,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:52:33,980 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 09:52:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 09:52:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 09:52:33,981 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 09:52:33,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:52:34,363 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m073465u-media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 09:52:34,382 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pvcuhayg-media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 09:52:34,391 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pysuamh9-media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 09:52:34,392 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jnyvuvr6-media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 09:52:35,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:39,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:39,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:52:41,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:43,996 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:52:47,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:47,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:52:47,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:52:47,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:52:49,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:49,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:52:49,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:52:49,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:52:52,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:54,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:52:54,012 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:52:57,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:52:59,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:52:59,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:52:59,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:52:59,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:02,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:02,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:02,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:03,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:04,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:04,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:08,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:08,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:08,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:08,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:08,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:09,851 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:53:13,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:14,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:14,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:17,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:17,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:17,121 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:19,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:19,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:19,794 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:19,794 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:19,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:24,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:24,066 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:24,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:29,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:30,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:30,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:30,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:30,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:32,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:32,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:32,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:34,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:35,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:39,852 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:53:40,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:40,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:40,735 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:40,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:40,738 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:44,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:44,286 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:46,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:48,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:53:48,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:53:48,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:53:51,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:53:52,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:53:52,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:53:52,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:53:52,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:53:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:53:54,296 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:53:57,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:02,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:54:02,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:54:02,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:54:02,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:54:03,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:03,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:03,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:03,997 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:04,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:54:04,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:54:06,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:54:06,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:54:06,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:54:06,734 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:54:08,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:09,853 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:54:13,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:14,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:54:14,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:54:18,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:19,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:19,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:19,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:24,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:29,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:34,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:34,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:34,204 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:34,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:39,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:39,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:54:44,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:49,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:54:49,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:54:49,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:54:50,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:54:55,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:00,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:00,850 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_3d170b31330953e821ba.png with policy now +2023-07-12 09:55:00,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,860 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_6f1a9a30ce0054e1f213.png with policy now +2023-07-12 09:55:00,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_a603ae3e847019132432.png with policy now +2023-07-12 09:55:00,879 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:55:00,880 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2360_4eb285dbb680a44d3c77.png with policy now +2023-07-12 09:55:00,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:00,922 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:00,922 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:00,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:01,163 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dg17jeso-media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 09:55:01,211 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xrd8kp3r-media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 09:55:01,214 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o4z0ey59-media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 09:55:01,215 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/btb36lsl-media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 09:55:04,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:04,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:55:04,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:04,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:04,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:04,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 09:55:04,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 09:55:04,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 09:55:04,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 09:55:04,619 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:55:04,976 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/useq9ui0-media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 09:55:04,978 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0khbs5bd-media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 09:55:04,989 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a09ls7b4-media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 09:55:04,993 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v1zi4ubv-media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 09:55:05,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:09,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:55:10,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:10,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:14,635 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:15,687 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:19,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:19,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:55:19,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:19,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:19,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:19,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:19,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:20,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:24,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:24,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:26,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:29,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:29,706 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:29,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:29,709 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:32,315 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:34,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:34,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:34,669 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:34,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:38,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:39,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:55:40,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:40,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:40,116 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:40,117 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:43,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:44,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:44,686 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:49,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:55:49,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:55:49,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:55:49,725 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:55:50,289 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:55:50,291 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:55:50,291 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:55:50,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:55:54,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:55:54,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:55:54,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:00,120 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:01,174 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:01,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:01,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:01,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:04,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:04,918 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:05,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:05,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:05,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:05,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:09,858 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:56:10,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:11,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:11,485 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:11,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:11,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:14,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:14,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:16,384 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:21,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:21,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:21,230 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:21,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:21,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:21,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:21,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:21,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:24,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:24,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:27,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:31,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:31,847 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:31,847 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:31,849 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:32,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:34,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:34,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:36,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:36,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:36,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:36,353 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:56:36,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:56:36,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:56:36,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:56:37,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:39,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:56:43,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:56:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:56:48,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:51,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:56:51,471 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:56:51,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:56:54,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:56:59,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:04,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:06,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:06,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:06,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:09,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:09,861 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:57:14,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:19,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:21,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:21,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:21,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:25,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:30,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:30,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_6b60edf29451c64b6e2b.png with policy now +2023-07-12 09:57:30,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_e3cc962dadab1208b51d.png with policy now +2023-07-12 09:57:30,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_4fd19c0f067294c7a9e6.png with policy now +2023-07-12 09:57:30,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 09:57:30,789 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2370_81faade70bf66d35a629.png with policy now +2023-07-12 09:57:30,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:57:30,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:57:30,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:57:30,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:57:31,080 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bhv1snwz-media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 09:57:31,081 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7d3651py-media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 09:57:31,119 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ygpr01p1-media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 09:57:31,256 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rkhebevp-media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 09:57:35,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:57:35,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:57:35,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 09:57:35,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 09:57:35,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 09:57:35,284 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 09:57:35,285 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 09:57:35,661 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1gheoe3b-media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 09:57:35,688 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tl2myosp-media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 09:57:35,692 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyb2dkpf-media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 09:57:35,694 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/88l6es9n-media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 09:57:35,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:36,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:36,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:36,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:39,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:57:40,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:57:41,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:45,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:57:47,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:50,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:57:50,604 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:57:50,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:57:50,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:57:52,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:57:52,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:57:52,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:57:52,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:57:55,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:57:55,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:57:57,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:00,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:00,980 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:00,981 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:00,982 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:02,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:05,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:05,339 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:07,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:07,789 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:08,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:09,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:58:10,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:10,504 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:10,505 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:10,507 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:13,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:15,356 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:15,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:18,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:20,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:20,746 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:20,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:20,748 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:22,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:22,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:24,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:25,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:25,373 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:29,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:30,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:30,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:30,903 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:30,905 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:34,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:35,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:35,590 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:38,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:58:38,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:38,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:39,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:58:40,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:41,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:41,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:41,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:41,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:45,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:45,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:45,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:51,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:58:51,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:58:51,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:58:51,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:58:51,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:58:53,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:58:53,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:58:53,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:58:55,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:58:55,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:58:57,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:02,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:59:02,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:59:02,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:59:02,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:59:03,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:05,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:59:05,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:59:06,762 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 09:59:06,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 09:59:06,765 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 09:59:06,767 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 09:59:08,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:08,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:08,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:08,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:09,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:59:13,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:15,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 09:59:15,654 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 09:59:18,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:23,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:23,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:23,619 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:24,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:29,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:34,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:38,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:38,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:38,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:39,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:39,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 09:59:44,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:49,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 09:59:53,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 09:59:53,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 09:59:53,644 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 09:59:55,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:00,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:01,410 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,410 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_841e12bb766b52954d05.png with policy now +2023-07-12 10:00:01,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,419 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_3b3663922cc96b138671.png with policy now +2023-07-12 10:00:01,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,429 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_f14220ee5227d3539481.png with policy now +2023-07-12 10:00:01,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:00:01,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2380_b3144dac275fe828dece.png with policy now +2023-07-12 10:00:01,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:01,564 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:01,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:01,566 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:01,717 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zn6xdbwo-media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 10:00:01,739 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/y4atd9sl-media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 10:00:01,741 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rk6fjqel-media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 10:00:01,830 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uxrazp87-media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 10:00:05,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:05,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:05,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 10:00:05,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 10:00:05,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 10:00:05,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 10:00:05,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:00:06,314 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9xihk05d-media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 10:00:06,331 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v2ksh9r5-media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 10:00:06,333 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2d1d8rw2-media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 10:00:06,341 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/46dzcq2k-media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 10:00:09,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:09,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:09,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:09,868 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:00:11,869 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:12,195 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:15,968 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:17,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:21,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:21,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:21,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:21,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:22,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:24,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:24,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:24,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:25,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:25,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:28,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:31,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:31,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:31,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:31,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:33,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:36,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:36,003 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:39,191 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:39,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:39,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:39,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:39,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:00:41,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:41,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:41,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:41,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:45,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:46,021 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:46,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:50,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:00:51,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:00:51,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:00:51,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:00:51,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:00:54,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:00:54,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:00:54,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:00:56,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:00:56,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:00:56,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:01,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:01,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:01,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:01,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:01,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:06,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:06,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:07,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:09,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:09,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:09,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:09,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:01:12,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:12,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:12,444 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:12,444 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:12,447 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:16,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:16,282 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:17,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:22,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:22,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:22,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:22,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:23,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:24,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:24,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:24,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:26,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:26,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:28,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:32,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:32,693 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:32,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:32,695 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:33,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:36,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:36,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:37,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:01:37,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:01:37,496 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:01:37,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:01:39,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:39,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:39,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:39,297 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:39,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:01:44,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:46,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:01:46,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:01:49,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:01:54,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:01:54,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:01:54,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:01:55,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:00,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:05,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:09,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:09,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:09,298 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:09,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:02:10,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:15,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:20,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:24,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:24,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:24,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:26,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:31,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:31,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,792 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_34c10cf3cd6d89963a05.png with policy now +2023-07-12 10:02:31,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_92717b0771dd961acad8.png with policy now +2023-07-12 10:02:31,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,884 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_674a3618e81973b0470a.png with policy now +2023-07-12 10:02:31,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:02:31,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2390_0c825ba6ed95b9246d26.png with policy now +2023-07-12 10:02:31,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:02:31,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:02:31,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:02:31,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:02:32,103 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wttkcjw5-media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 10:02:32,199 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ryuwmiw-media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 10:02:32,209 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27tzb0cd-media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 10:02:32,228 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r3v6r8lm-media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 10:02:36,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:02:36,615 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:02:36,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 10:02:36,616 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 10:02:36,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 10:02:36,617 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 10:02:36,618 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:02:36,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:36,968 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ptmgkva1-media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 10:02:37,020 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ou9mjgek-media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 10:02:37,021 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wd5990ky-media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 10:02:37,046 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpguz4xa-media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 10:02:39,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:39,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:39,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:39,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:02:41,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:02:42,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:46,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:02:47,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:50,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:02:50,879 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:02:50,879 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:02:50,881 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:02:52,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:02:54,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:02:54,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:02:54,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:02:56,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:02:56,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:02:58,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:01,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:01,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:01,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:01,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:04,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:06,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:06,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:09,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:09,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:09,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:09,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:03:09,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:11,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:11,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:11,095 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:11,097 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:15,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:16,684 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:16,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:21,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:21,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:21,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:21,098 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:21,100 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:24,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:24,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:24,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:26,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:26,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:26,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:31,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:31,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:31,300 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:31,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:31,303 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:36,449 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:36,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:36,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:39,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:39,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:39,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:39,876 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:03:41,141 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:41,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:41,142 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:41,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:42,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:46,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:46,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:47,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:51,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:03:51,249 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:03:51,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:03:51,252 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:03:53,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:03:54,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:03:54,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:03:54,347 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:03:56,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:03:56,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:03:58,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:01,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:04:01,454 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:04:01,454 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:04:01,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:04:03,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:05,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:04:05,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:04:05,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:04:05,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:04:06,983 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:04:06,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:04:08,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:09,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:09,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:09,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:09,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:04:13,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:17,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:04:18,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:23,880 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:24,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:24,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:24,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:29,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:34,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:39,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:39,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:39,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:39,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:04:39,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:44,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:49,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:04:54,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:04:54,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:04:54,409 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:04:55,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:00,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:00,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_7fcaf32b847afe6bc2ca.png with policy now +2023-07-12 10:05:00,688 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_3cef30f8c2574b5c86c0.png with policy now +2023-07-12 10:05:00,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,699 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png with policy now +2023-07-12 10:05:00,709 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:05:00,709 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2400_a254fe937a4ee82dda50.png with policy now +2023-07-12 10:05:00,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:00,749 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:00,749 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:00,751 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:00,927 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i3o54obv-media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 10:05:01,003 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5o8cho5b-media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 10:05:01,014 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q0cxw1c8-media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 10:05:01,015 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/46qvkwex-media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 10:05:06,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:07,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:07,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:07,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 10:05:07,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 10:05:07,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 10:05:07,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 10:05:07,302 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:05:07,689 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qqeeham1-media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 10:05:07,690 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cmla0q6e-media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 10:05:07,722 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t3edz0ov-media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 10:05:07,752 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/t8li6o3u-media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 10:05:09,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:09,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:09,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:05:11,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:12,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:17,317 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:17,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:20,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:20,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:20,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:20,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:22,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:24,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:05:24,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:24,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:27,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:27,334 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:28,644 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:30,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:30,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:30,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:30,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:34,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:37,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:37,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:39,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:05:39,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:39,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:39,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:39,881 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:05:40,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:40,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:40,032 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:40,033 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:44,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:47,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:47,368 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:49,710 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:49,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:49,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:49,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:05:50,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:54,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:05:54,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:05:54,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:05:55,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:05:57,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:05:57,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:05:59,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:05:59,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:05:59,737 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:05:59,739 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:01,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:07,085 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:07,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:07,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:09,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:09,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:09,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:09,882 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:06:10,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:10,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:10,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:10,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:12,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:17,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:17,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:17,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:19,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:19,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:19,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:19,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:22,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:24,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:24,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:24,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:27,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:27,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:28,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:30,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:30,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:30,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:30,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:34,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:35,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:06:35,240 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:06:35,240 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:06:35,242 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:06:37,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:06:37,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:06:39,883 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:06:39,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:39,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:39,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:39,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:45,030 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:50,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:06:54,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:06:54,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:06:54,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:06:56,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:01,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:06,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:09,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:07:09,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:09,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:09,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:12,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:17,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:22,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:24,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:25,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:25,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:27,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:29,845 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,845 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_f055f10be828e22bd53d.png with policy now +2023-07-12 10:07:29,854 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,854 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_fcac0499a5c4fff7482f.png with policy now +2023-07-12 10:07:29,865 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_5eb323fa89395b0ed41c.png with policy now +2023-07-12 10:07:29,875 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:07:29,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2410_5870baca82c8f1ea9c65.png with policy now +2023-07-12 10:07:29,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:29,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:07:29,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:07:29,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:07:30,183 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dp1ixhi3-media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 10:07:30,188 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fsaekjnv-media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 10:07:30,192 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/egl4orij-media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 10:07:30,298 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ck4mh8lu-media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 10:07:32,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 10:07:37,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 10:07:37,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 10:07:37,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:07:37,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:38,326 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lspb0yyj-media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 10:07:38,349 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yq5v6xcv-media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 10:07:38,443 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wxkn6c5q-media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 10:07:38,549 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r34l3gke-media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 10:07:39,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:07:40,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:40,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:40,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:40,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:43,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:47,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:07:48,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:49,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:49,285 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:07:49,285 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:07:49,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:07:54,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:07:55,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:07:55,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:07:55,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:07:58,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:07:58,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:07:58,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:07:58,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:07:58,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:07:58,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:07:59,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:05,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:08,027 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:08,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:08,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:08,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:08,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:08,967 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:09,887 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:08:10,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:10,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:10,163 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:11,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:17,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:18,044 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:18,045 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:18,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:18,802 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:18,802 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:18,804 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:22,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:26,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:26,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:26,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:28,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:28,062 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:28,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:29,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:29,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:29,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:29,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:33,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:38,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:38,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:39,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:39,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:08:40,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:40,272 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:40,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:40,274 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:41,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:41,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:41,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:45,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:48,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:48,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:08:50,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:08:50,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:08:50,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:08:50,320 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:08:51,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:56,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:08:56,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:08:56,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:08:57,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:08:58,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:08:58,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:09:01,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:09:01,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:09:01,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:09:01,024 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:09:03,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:05,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:09:05,873 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:09:05,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:09:05,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:09:08,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:09:08,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:09:08,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:09,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:09:11,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:11,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:11,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:09:14,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:19,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:24,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:26,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:26,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:26,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:09:30,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:35,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:39,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:09:40,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:41,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:41,254 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:41,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:09:46,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:51,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:56,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:09:56,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:09:56,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:09:56,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:00,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,169 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_e300a087119470bd19f7.png with policy now +2023-07-12 10:10:00,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,178 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_a3ef41aa53f7542e2ff6.png with policy now +2023-07-12 10:10:00,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_86a10e9e981b83c54181.png with policy now +2023-07-12 10:10:00,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:10:00,198 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2420_508d57a7914d9268a798.png with policy now +2023-07-12 10:10:00,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:00,238 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:00,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:00,240 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:00,523 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qnrnjmao-media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 10:10:00,532 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u01fd0md-media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 10:10:00,538 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jdmdtbpw-media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 10:10:00,545 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bc1an829-media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 10:10:02,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:07,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:08,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:08,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:08,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 10:10:08,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 10:10:08,651 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 10:10:08,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 10:10:08,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:10:09,047 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xn35i7dg-media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 10:10:09,051 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xxqyv611-media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 10:10:09,061 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x97hkk2m-media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 10:10:09,086 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/covzlcdh-media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 10:10:09,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:10:10,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:11,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:11,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:11,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:12,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:17,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:18,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:19,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:19,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:19,815 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:19,817 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:23,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:27,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:27,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:27,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:28,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:28,685 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:28,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:29,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:29,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:29,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:29,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:34,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:38,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:38,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:39,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:39,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:10:40,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:40,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:40,048 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:40,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:42,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:42,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:42,151 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:45,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:48,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:48,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:10:49,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:10:49,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:10:49,949 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:10:49,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:10:50,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:56,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:10:57,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:10:57,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:10:57,624 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:10:58,735 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:10:58,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:00,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:00,820 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:00,820 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:00,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:01,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:07,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:08,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:08,982 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:09,894 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:11:10,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:10,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:10,995 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:10,997 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:12,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:12,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:12,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:11:12,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:18,025 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:19,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:19,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:21,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:21,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:21,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:21,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:23,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:27,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:27,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:27,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:11:28,724 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:29,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:29,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:31,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:31,350 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:31,350 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:31,352 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:34,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:36,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:11:36,484 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:11:36,485 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:11:36,487 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:11:39,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:11:39,032 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:11:39,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:11:39,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:42,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:42,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:42,648 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:11:45,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:50,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:55,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:11:57,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:11:57,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:11:57,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:00,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:05,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:09,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:12:10,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:12,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:12,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:12,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:16,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:21,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:26,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:27,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:27,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:27,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:30,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,149 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_d23160faa6354332ac90.png with policy now +2023-07-12 10:12:30,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_e3cbf88a5d454b013c98.png with policy now +2023-07-12 10:12:30,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,167 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_bdfc2304f95d55e3c519.png with policy now +2023-07-12 10:12:30,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:12:30,177 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2430_9d7fc07bee22787a2007.png with policy now +2023-07-12 10:12:30,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:30,221 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:12:30,221 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:12:30,223 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:12:30,484 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5egvasl1-media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 10:12:30,493 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9w1h2neg-media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 10:12:30,553 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vd6lcxqk-media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 10:12:30,556 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2flwhrxk-media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 10:12:32,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:37,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:39,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:12:39,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:12:39,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 10:12:39,345 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 10:12:39,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 10:12:39,346 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 10:12:39,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:12:39,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:39,745 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u5nk2mcf-media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 10:12:39,759 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/60vzgurn-media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 10:12:39,774 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a3r605hr-media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 10:12:39,850 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hvn9klte-media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 10:12:39,898 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:12:42,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:42,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:42,744 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:42,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:48,442 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:48,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:48,738 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:12:48,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:12:48,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:12:49,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:12:49,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:12:53,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:12:57,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:12:57,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:12:57,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:12:58,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:12:58,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:12:58,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:12:58,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:12:59,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:12:59,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:12:59,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:05,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:07,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:07,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:07,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:07,850 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:09,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:09,393 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:09,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:13:10,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:13,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:13,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:13,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:15,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:19,410 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:19,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:19,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:19,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:19,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:21,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:26,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:29,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:29,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:29,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:29,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:29,427 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:29,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:29,587 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:29,587 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:29,589 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:31,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:36,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:39,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:39,652 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:39,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:13:40,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:40,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:40,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:40,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:42,057 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:44,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:44,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:44,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:47,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:49,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:49,668 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:13:50,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:13:50,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:13:50,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:13:50,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:13:52,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:57,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:13:59,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:13:59,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:13:59,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:13:59,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:13:59,682 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:14:00,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:14:00,534 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:14:00,535 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:14:00,537 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:14:03,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:05,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:14:05,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:14:05,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:14:05,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:14:08,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:09,698 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:14:09,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:14:09,901 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:14:13,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:14,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:14,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:14,602 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:19,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:24,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:29,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:29,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:29,603 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:30,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:35,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:39,902 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:14:40,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:44,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:46,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:51,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:56,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:14:59,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:14:59,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:14:59,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:14:59,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,819 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_f15d4e86eefcf39e1eae.png with policy now +2023-07-12 10:14:59,829 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_8826c1292b54e84dc27b.png with policy now +2023-07-12 10:14:59,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_02cad4f86332c4d30e0f.png with policy now +2023-07-12 10:14:59,870 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:14:59,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2440_b90423e76bb46ed0bd51.png with policy now +2023-07-12 10:14:59,888 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:14:59,890 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:14:59,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:14:59,895 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:00,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:15:00,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 10:15:00,006 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 10:15:00,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 10:15:00,007 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 10:15:00,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:15:00,140 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pakh2qdm-media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 10:15:00,150 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zwb74alu-media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 10:15:00,200 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ug0k2e8w-media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 10:15:00,207 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ih8ew1uf-media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 10:15:00,560 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/989qnze8-media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 10:15:00,777 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ufxvj7v-media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 10:15:00,900 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhkwbe7n-media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 10:15:01,135 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ihzn3xgn-media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 10:15:01,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:06,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:09,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:09,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:15:10,023 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:11,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:14,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:14,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:14,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:15:17,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:19,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:19,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:19,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:19,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:20,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:15:20,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:23,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:28,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:29,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:29,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:29,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:15:30,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:30,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:30,251 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:30,252 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:30,254 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:33,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:38,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:39,905 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:15:39,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:39,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:39,998 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:40,000 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:40,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:15:40,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:43,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:44,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:44,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:15:48,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:50,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:15:50,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:15:50,338 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:15:50,338 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:15:50,340 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:15:54,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:59,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:15:59,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:15:59,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:15:59,999 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:00,318 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:00,319 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:00,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:00,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:00,638 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:00,640 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:05,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:09,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:16:10,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:10,336 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:10,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:10,727 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:10,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:10,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:10,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:15,268 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:16:15,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:16:15,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:15,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:20,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:20,387 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:20,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:20,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:20,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:20,632 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:21,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:26,711 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:30,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:16:30,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:16:30,269 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:30,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:30,403 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:30,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:30,823 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:30,823 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:30,825 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:31,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:35,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:16:35,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:16:35,303 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:16:35,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:16:36,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:39,907 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:16:40,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:16:40,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:16:41,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:45,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:16:45,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:16:45,288 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:16:47,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:52,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:16:57,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:00,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:00,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:00,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:02,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:07,525 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:09,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:17:12,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:15,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:15,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:15,529 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:18,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:23,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:28,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:29,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,377 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_f4dd09917aefe94149cf.png with policy now +2023-07-12 10:17:29,438 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_807425807eddcc0e0aaa.png with policy now +2023-07-12 10:17:29,450 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_30096936a6475e462a39.png with policy now +2023-07-12 10:17:29,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:17:29,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2450_8fe94ed8c310dae4cd3c.png with policy now +2023-07-12 10:17:29,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:29,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:17:29,502 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:17:29,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:17:29,745 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/87puz1sm-media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 10:17:29,817 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xbey8ss3-media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 10:17:29,846 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gkt6ivub-media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 10:17:29,855 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mp4bphr3-media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 10:17:30,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:30,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:30,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:17:30,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 10:17:30,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 10:17:30,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 10:17:30,722 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 10:17:30,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:17:31,068 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3vxidlgg-media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 10:17:31,080 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u09nhyhp-media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 10:17:31,084 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z1f7zqo9-media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 10:17:31,085 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cpl48eo2-media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 10:17:33,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:39,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:39,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:39,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:17:40,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:17:44,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:46,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:17:46,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:17:46,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:17:48,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:48,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:17:48,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:17:48,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:17:49,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:50,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:17:50,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:17:55,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:17:59,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:17:59,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:17:59,011 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:17:59,013 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:00,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:00,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:01,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:01,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:01,230 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:01,231 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:06,032 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:09,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:09,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:09,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:09,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:09,911 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:18:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:10,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:11,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:16,786 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:16,787 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:16,787 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:17,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:19,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:19,792 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:19,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:19,794 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:20,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:20,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:23,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:28,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:30,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:30,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:30,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:30,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:31,058 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:31,059 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:31,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:31,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:31,912 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:33,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:39,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:39,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:39,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:39,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:39,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:39,912 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:18:41,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:41,076 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:44,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:47,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:18:47,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:18:47,051 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:18:49,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:49,512 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:49,512 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:49,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:18:50,515 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:51,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:18:51,092 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:18:55,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:18:59,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:18:59,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:18:59,711 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:18:59,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:19:00,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:01,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:19:01,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:19:02,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:02,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:04,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:19:04,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:19:04,123 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:19:04,125 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:19:05,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:09,913 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:19:10,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:11,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:19:11,125 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:19:15,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:17,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:19:17,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:17,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:21,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:26,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:31,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:32,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:19:32,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:32,329 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:36,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:39,914 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:19:41,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:46,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:47,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:19:47,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:19:47,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:19:52,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:57,583 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:19:58,665 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,665 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_d5445284771d7b2c66a0.png with policy now +2023-07-12 10:19:58,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_c867e6da32e13d963e59.png with policy now +2023-07-12 10:19:58,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_815ded56ecd664726052.png with policy now +2023-07-12 10:19:58,717 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:19:58,717 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2460_e5fd4b41b8c0c33c93ac.png with policy now +2023-07-12 10:19:58,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:19:58,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:19:58,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:19:58,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:19:58,972 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d91som57-media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 10:19:59,005 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z4s5lui1-media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 10:19:59,041 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xd0ug1by-media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 10:19:59,124 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jqtvn2pt-media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 10:20:01,429 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:01,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 10:20:01,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 10:20:01,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 10:20:01,431 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 10:20:01,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:20:01,773 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vmpfkiwa-media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 10:20:01,779 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pnjz9juk-media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 10:20:01,783 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/skyxn26h-media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 10:20:01,797 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qum8izu1-media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 10:20:02,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:02,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:02,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:03,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:08,882 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:08,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:09,915 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:20:11,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:14,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:17,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:17,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:17,652 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:18,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:18,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:18,294 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:18,297 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:20,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:21,464 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:21,465 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:26,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:27,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:27,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:27,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:27,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:31,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:31,482 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:32,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:32,684 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:32,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:32,686 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:37,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:37,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:37,815 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:37,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:37,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:39,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:20:41,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:41,500 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:42,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:47,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:20:47,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:20:47,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:20:47,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:48,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:48,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:48,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:48,480 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:20:51,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:20:51,515 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:20:53,497 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:58,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:20:58,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:20:58,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:20:58,944 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:20:58,946 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:01,739 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:02,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:21:02,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:02,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:03,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:09,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:09,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:21:10,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:10,133 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:10,133 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:10,135 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:11,759 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:14,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:18,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:21:18,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:18,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:20,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:20,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:20,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:20,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:20,701 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:21,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:21,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:25,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:30,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:30,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:30,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:30,929 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:30,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:31,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:31,793 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:33,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:33,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:35,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:21:35,276 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:21:35,276 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:21:35,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:21:36,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:39,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:21:41,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:21:41,809 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:21:41,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:46,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:48,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:21:48,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:21:48,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:21:52,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:21:57,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:02,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:03,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:03,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:03,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:08,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:09,920 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:22:13,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:18,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:18,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:18,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:19,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:24,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:29,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:29,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,703 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_edd32e44eaf81373d115.png with policy now +2023-07-12 10:22:29,712 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_db2b2e82dd30d35d1311.png with policy now +2023-07-12 10:22:29,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,722 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_8dcd67f2db2d9902e40b.png with policy now +2023-07-12 10:22:29,732 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:22:29,732 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2470_97ab1f90f62f7fd81135.png with policy now +2023-07-12 10:22:29,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:29,773 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:22:29,773 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:22:29,775 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:22:30,155 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8baaxgt7-media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 10:22:30,164 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9414n9xa-media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 10:22:30,177 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bwktdb8q-media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 10:22:30,209 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gmw3sgmp-media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 10:22:32,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:22:32,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 10:22:32,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 10:22:32,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 10:22:32,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 10:22:32,108 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:22:32,530 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g9tz651v-media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 10:22:32,532 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wvt5wsen-media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 10:22:32,548 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g2roygn9-media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 10:22:32,671 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xiexvg7o-media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 10:22:33,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:33,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:33,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:35,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:39,921 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:22:40,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:40,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:42,123 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:22:46,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:22:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:22:48,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:22:50,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:50,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:22:50,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:22:50,045 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:22:52,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:52,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:22:52,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:22:57,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:22:58,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:22:58,910 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:22:58,910 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:22:58,912 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:02,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:02,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:03,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:03,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:03,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:03,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:08,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:09,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:09,368 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:09,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:09,370 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:09,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:23:12,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:12,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:13,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:18,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:18,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:18,846 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:18,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:19,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:19,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:19,412 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:19,414 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:22,196 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:23,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:29,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:29,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:29,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:29,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:29,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:32,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:32,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:34,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:34,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:34,520 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:34,520 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:39,924 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:23:39,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:40,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:40,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:40,061 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:40,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:42,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:42,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:45,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:51,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:23:51,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:23:51,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:23:51,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:23:51,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:23:51,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:23:51,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:23:51,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:23:52,474 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:23:52,475 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:23:56,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:01,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:24:01,728 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:01,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:24:01,729 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:24:01,731 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:24:02,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:24:02,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:24:06,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:06,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:06,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:06,312 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:24:06,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:24:06,377 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:24:06,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:24:06,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:09,925 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:24:11,926 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:12,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:24:12,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:24:16,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:21,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:21,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:21,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:22,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:27,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:32,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:36,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:36,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:36,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:37,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:39,926 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:24:42,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:47,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:51,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:24:51,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:24:51,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:24:53,380 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:24:58,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:00,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,605 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_b53f3dded240b500b1cb.png with policy now +2023-07-12 10:25:00,614 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,614 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_a061712ad948dbaccbd5.png with policy now +2023-07-12 10:25:00,624 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_129f224b0015451a7862.png with policy now +2023-07-12 10:25:00,634 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:25:00,634 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2480_6c96769927607cfa02a7.png with policy now +2023-07-12 10:25:00,678 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:00,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:00,679 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:00,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:00,907 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ogauxbch-media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 10:25:00,935 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6f6z8bee-media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 10:25:00,942 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/02rrpi4j-media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 10:25:00,956 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8o2y4t2x-media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 10:25:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:02,816 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 10:25:02,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 10:25:02,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 10:25:02,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 10:25:02,818 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:25:03,195 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c8qnj3dd-media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 10:25:03,209 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lkzltoas-media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 10:25:03,249 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kajg4clw-media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 10:25:03,296 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tmg94lwf-media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 10:25:03,891 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:06,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:06,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:06,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:09,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:09,927 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:25:10,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:12,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:14,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:19,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:19,247 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:19,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:19,249 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:20,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:21,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:21,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:21,274 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:22,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:22,851 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:26,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:29,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:29,708 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:29,708 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:29,711 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:31,712 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:32,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:32,867 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:36,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:36,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:36,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:38,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:39,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:39,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:39,448 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:39,450 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:39,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:25:42,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:42,885 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:44,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:49,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:25:49,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:25:49,694 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:25:49,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:25:49,697 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:25:51,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:25:51,666 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:25:51,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:25:52,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:25:52,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:25:54,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:00,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:00,329 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:00,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:00,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:00,332 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:03,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:03,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:05,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:06,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:06,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:09,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:26:10,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:11,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:11,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:11,114 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:11,116 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:13,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:13,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:16,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:21,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:21,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:21,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:21,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:21,645 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:21,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:21,666 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:23,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:23,168 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:26,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:31,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:31,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:31,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:31,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:31,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:33,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:33,185 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:36,626 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:36,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:26:36,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:36,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:36,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:26:36,793 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:26:36,793 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:26:36,795 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:26:39,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:26:41,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:43,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:26:43,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:26:46,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:51,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:26:51,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:26:51,762 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:26:52,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:26:57,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:02,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:06,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:06,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:06,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:07,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:09,932 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:27:12,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:17,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:21,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:21,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:21,873 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:23,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:28,919 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:30,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:30,985 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_e2238e57e86ad9eb1236.png with policy now +2023-07-12 10:27:31,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:31,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_fbe171c1140c25cd1cb2.png with policy now +2023-07-12 10:27:31,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:31,091 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_eb52c1f3852b2db713d6.png with policy now +2023-07-12 10:27:31,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:27:31,101 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2490_fb4911582a74ff6e59f1.png with policy now +2023-07-12 10:27:31,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:31,143 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:27:31,144 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:27:31,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:27:31,319 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5an5exb6-media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 10:27:31,398 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/27hpj0zf-media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 10:27:31,403 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rxflr06c-media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 10:27:31,644 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pzzoypxh-media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 10:27:33,516 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:27:33,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 10:27:33,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 10:27:33,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 10:27:33,518 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 10:27:33,519 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:27:34,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:34,724 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8duytq7b-media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 10:27:34,854 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yroi7zq6-media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 10:27:35,107 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gffecb8v-media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 10:27:35,281 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nyszv03s-media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 10:27:36,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:36,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:36,830 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:39,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:27:39,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:41,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:43,533 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:27:45,768 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:50,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:50,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:27:50,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:27:50,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:27:51,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:52,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:27:52,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:27:52,527 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:27:54,551 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:27:54,552 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:27:57,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:27:59,676 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:27:59,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:27:59,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:27:59,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:02,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:05,567 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:05,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:07,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:07,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:07,669 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:07,738 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:09,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:28:10,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:10,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:10,663 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:10,664 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:13,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:16,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:16,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:19,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:20,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:20,705 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:20,705 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:20,707 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:23,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:23,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:23,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:24,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:27,637 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:27,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:30,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:31,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:31,826 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:31,826 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:31,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:36,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:38,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:38,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:38,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:38,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:38,883 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:39,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:28:41,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:41,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:41,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:41,677 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:41,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:46,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:49,900 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:28:49,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:28:51,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:28:51,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:28:51,886 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:28:51,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:28:51,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:28:53,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:28:53,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:28:53,727 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:28:57,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:00,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:29:00,919 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:29:01,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:29:01,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:29:01,824 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:29:01,826 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:29:02,827 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:06,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:29:06,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:29:06,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:29:06,393 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:29:08,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:08,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:08,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:08,912 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:09,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:29:11,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:29:11,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:29:13,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:18,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:23,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:24,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:24,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:24,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:29,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:34,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:39,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:39,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:39,131 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:39,936 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:29:39,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:44,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:49,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:29:54,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:29:54,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:29:54,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:29:55,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:00,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:01,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,093 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_1a8029b03d2f4e8ba6b5.png with policy now +2023-07-12 10:30:01,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_a3765156a53105d7a44f.png with policy now +2023-07-12 10:30:01,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,115 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_7b6d1552c5731ce4d99a.png with policy now +2023-07-12 10:30:01,197 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:30:01,197 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2500_45f174c3353c67e7d2fa.png with policy now +2023-07-12 10:30:01,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:01,239 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:01,239 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:01,241 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:01,428 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tuyutaup-media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 10:30:01,434 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/httc3xuk-media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 10:30:01,454 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbvtqk5b-media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 10:30:01,523 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpg6z62d-media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 10:30:06,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:07,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:07,242 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:07,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 10:30:07,243 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 10:30:07,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 10:30:07,244 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 10:30:07,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:30:07,607 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uniypzz0-media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 10:30:07,614 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/geip8mmw-media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 10:30:07,618 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bfa3yq8w-media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 10:30:07,619 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vyeq0c04-media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 10:30:09,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:30:09,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:09,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:09,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:30:11,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:11,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:17,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:18,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:20,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:20,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:20,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:20,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:22,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:24,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:30:24,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:24,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:27,774 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:29,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:29,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:30,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:30,682 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:30,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:30,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:33,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:38,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:39,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:39,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:30:39,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:39,939 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:30:40,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:40,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:41,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:41,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:41,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:41,612 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:44,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:49,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:30:51,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:30:51,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:30:52,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:30:52,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:30:52,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:30:52,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:30:54,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:30:54,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:30:55,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:00,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:02,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:02,329 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:02,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:02,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:02,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:02,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:06,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:09,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:09,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:09,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:31:11,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:12,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:12,703 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:12,703 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:12,706 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:13,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:13,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:16,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:22,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:22,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:22,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:22,698 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:22,700 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:24,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:24,574 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:24,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:24,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:24,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:27,454 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:32,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:32,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:32,916 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:32,917 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:32,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:35,591 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:35,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:37,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:31:37,413 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:31:37,414 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:31:37,416 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:31:37,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:39,941 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:31:39,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:39,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:39,956 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:43,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:46,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:31:46,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:31:48,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:53,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:31:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:31:55,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:31:55,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:31:58,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:03,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:08,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:09,943 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:32:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:10,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:32:10,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:14,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:19,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:24,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:25,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:25,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:29,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:31,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_f73a3d21c0ad929cc0df.png with policy now +2023-07-12 10:32:31,702 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,702 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png with policy now +2023-07-12 10:32:31,713 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,713 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_2345ba38c318660ea20d.png with policy now +2023-07-12 10:32:31,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:32:31,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2510_4acac46bf9cdc8077cd8.png with policy now +2023-07-12 10:32:31,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:32:31,765 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:32:31,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:32:31,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:32:32,049 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dpb15br1-media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 10:32:32,068 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdnyah1p-media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 10:32:32,079 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/msq3ibcj-media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 10:32:32,135 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f5ws0o1d-media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 10:32:34,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:39,944 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:32:39,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:40,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:32:40,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:40,162 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:41,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:32:41,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:32:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:32:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 10:32:41,934 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 10:32:41,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 10:32:41,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 10:32:41,935 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:32:42,461 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z2g9q0ur-media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 10:32:42,510 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x0poqlfz-media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 10:32:42,539 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ws74ovw8-media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 10:32:42,564 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpd2fk6g-media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 10:32:45,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:50,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:32:50,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:32:50,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:32:50,946 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:32:50,948 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:32:52,951 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:32:52,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:32:55,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:32:55,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:32:55,173 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:32:55,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:01,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:01,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:01,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:01,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:01,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:03,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:03,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:06,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:09,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:33:10,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:10,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:10,330 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:10,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:10,895 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:10,896 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:10,898 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:11,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:14,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:14,988 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:16,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:20,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:20,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:20,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:20,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:22,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:25,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:25,648 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:25,649 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:26,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:26,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:28,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:31,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:31,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:31,578 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:31,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:33,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:38,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:39,946 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:33:40,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:40,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:40,835 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:41,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:41,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:41,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:41,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:44,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:48,253 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:48,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:49,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:52,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:33:52,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:33:52,748 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:33:52,750 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:33:54,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:33:55,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:33:55,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:33:55,835 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:33:59,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:33:59,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:33:59,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:02,817 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:34:02,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:34:02,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:34:02,820 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:34:04,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:07,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:34:07,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:34:07,184 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:34:07,186 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:34:09,947 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:34:09,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:10,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:34:10,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:34:10,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:10,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:10,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:15,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:20,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:25,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:25,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:25,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:25,974 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:31,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:36,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:39,948 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:34:41,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:41,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:41,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:41,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:46,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:51,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:34:56,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:34:56,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:34:56,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:34:56,217 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:01,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:01,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_ce5f5d922b12cb9a8e58.png with policy now +2023-07-12 10:35:01,386 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_55683596f533718ecbc7.png with policy now +2023-07-12 10:35:01,495 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,495 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_9cd53a6492dff1119fcd.png with policy now +2023-07-12 10:35:01,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:35:01,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2520_4e5b7eac0474303b7db6.png with policy now +2023-07-12 10:35:01,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:01,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:01,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:01,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:01,694 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/45b1v69l-media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 10:35:01,714 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ver8pl49-media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 10:35:01,813 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7cwq2ety-media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 10:35:01,829 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vshm5jod-media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 10:35:05,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:05,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 10:35:05,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 10:35:05,597 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 10:35:05,598 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 10:35:05,598 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:35:05,975 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/105jvs7p-media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 10:35:05,977 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uh6m09pp-media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 10:35:05,978 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5ktjowrr-media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 10:35:05,981 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/etlcnkv7-media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 10:35:06,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:09,949 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:35:11,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:11,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:11,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:35:11,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:11,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:16,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:17,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:20,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:20,606 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:20,607 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:20,609 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:22,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:26,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:26,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:26,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:35:27,629 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:27,630 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:28,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:31,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:31,013 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:31,013 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:31,015 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:33,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:38,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:38,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:38,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:39,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:35:40,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:41,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:41,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:41,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:41,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:41,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:41,150 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:35:44,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:49,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:35:49,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:35:49,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:51,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:35:51,453 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:35:51,453 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:35:51,456 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:35:54,894 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:35:56,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:35:56,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:35:56,334 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:00,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:00,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:00,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:01,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:01,571 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:01,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:01,573 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:05,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:09,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:36:10,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:11,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:11,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:11,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:11,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:11,921 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:12,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:12,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:12,145 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:12,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:16,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:22,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:22,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:22,509 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:22,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:22,511 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:22,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:22,938 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:26,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:26,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:26,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:28,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:31,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:31,862 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:31,862 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:31,864 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:33,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:36,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:36:36,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:36:36,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:36:36,501 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:36:39,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:39,952 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:36:41,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:41,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:41,612 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:44,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:44,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:36:44,971 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:36:49,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:54,740 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:36:56,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:36:56,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:36:56,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:36:59,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:04,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:09,953 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:37:10,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:11,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:11,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:11,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:16,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:21,199 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:26,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:26,865 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:30,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,899 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_a5bdcb2b01b23f824fcf.png with policy now +2023-07-12 10:37:30,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,908 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_e419bc8806d8bdc57fa0.png with policy now +2023-07-12 10:37:30,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,918 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_48e39b87d59cc06724b3.png with policy now +2023-07-12 10:37:30,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:37:30,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2530_4436a3e5cc352d2a1017.png with policy now +2023-07-12 10:37:30,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:30,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:37:30,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:37:30,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:37:31,494 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bdkiyd14-media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 10:37:31,604 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jpqsbzbg-media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 10:37:31,752 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eni5dlxq-media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 10:37:31,797 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5p88qyx4-media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 10:37:31,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:37,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:39,954 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:37:40,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:37:40,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:37:40,271 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 10:37:40,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 10:37:40,272 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 10:37:40,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 10:37:40,274 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:37:40,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:40,763 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2mk7nq4t-media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 10:37:40,771 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ktv84ge-media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 10:37:40,774 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xl7q67at-media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 10:37:40,788 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8e2r2crs-media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 10:37:42,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:42,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:42,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:42,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:48,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:50,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:50,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:37:50,676 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:37:50,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:37:51,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:37:51,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:37:53,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:57,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:37:57,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:37:57,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:37:59,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:37:59,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:37:59,754 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:37:59,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:37:59,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:02,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:02,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:04,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:09,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:09,332 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:09,333 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:09,334 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:09,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:38:09,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:12,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:12,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:12,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:13,325 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:13,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:15,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:19,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:19,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:19,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:19,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:20,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:24,343 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:24,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:26,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:27,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:27,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:27,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:31,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:31,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:31,047 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:31,049 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:32,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:35,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:35,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:37,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:39,956 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:38:42,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:42,097 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:42,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:42,100 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:42,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:42,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:42,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:43,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:46,610 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:46,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:49,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:52,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:38:52,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:38:52,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:38:52,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:38:54,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:38:57,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:38:57,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:38:57,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:38:57,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:38:57,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:38:59,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:02,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:39:02,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:39:02,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:39:02,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:39:05,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:07,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:39:07,301 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:39:07,301 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:39:07,484 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:39:08,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:39:08,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:39:09,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:39:10,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:12,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:12,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:12,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:39:16,747 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:19,663 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:39:21,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:26,749 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:27,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:27,700 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:39:31,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:36,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:39,958 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:39:41,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:42,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:42,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:42,701 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:39:47,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:52,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:57,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:39:57,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:39:57,851 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:39:57,851 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:01,673 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,674 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_03d4142985e8dc5f61fc.png with policy now +2023-07-12 10:40:01,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,684 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_13c9cc17dc94ed86351c.png with policy now +2023-07-12 10:40:01,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_97a6019e1743b1256999.png with policy now +2023-07-12 10:40:01,783 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:40:01,783 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2540_1248be5f76e40ac7db9c.png with policy now +2023-07-12 10:40:01,826 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:01,827 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:01,827 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:01,829 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:02,040 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tpbc2rn0-media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 10:40:02,041 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e121qmzs-media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 10:40:02,143 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9yttxo7a-media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 10:40:02,165 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vlw43936-media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 10:40:02,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:03,963 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:03,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 10:40:03,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 10:40:03,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 10:40:03,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 10:40:03,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:40:04,331 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uyi9il8b-media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 10:40:04,334 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6e9akvce-media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 10:40:04,346 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c55ru7mk-media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 10:40:04,420 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tm4v2t8t-media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 10:40:07,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:09,959 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:40:11,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:13,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:13,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:13,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:13,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:14,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:18,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:20,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:20,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:20,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:20,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:24,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:26,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:26,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:28,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:28,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:28,311 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:29,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:30,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:30,292 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:30,293 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:30,295 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:34,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:37,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:37,020 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:39,960 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:40:39,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:40,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:40,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:40,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:40,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:43,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:43,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:43,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:45,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:48,037 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:48,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:40:50,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:50,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:40:50,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:40:50,945 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:40:50,947 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:40:55,832 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:40:58,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:40:58,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:40:58,487 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:40:59,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:40:59,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:01,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:01,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:01,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:01,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:01,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:06,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:09,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:41:10,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:10,298 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:11,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:11,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:11,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:11,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:12,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:13,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:13,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:13,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:41:17,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:21,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:21,316 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:21,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:21,989 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:21,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:21,991 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:22,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:28,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:28,591 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:28,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:28,592 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:41:31,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:31,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:31,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:31,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:32,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:32,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:33,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:36,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:41:36,542 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:41:36,542 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:41:36,544 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:41:39,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:39,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:41:43,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:41:43,352 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:41:43,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:43,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:43,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:41:44,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:49,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:41:58,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:41:58,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:41:58,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:00,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:05,776 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:09,963 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:42:10,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:13,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:13,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:13,719 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:16,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:21,792 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:26,793 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:28,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:28,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:31,031 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,031 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_8cdd64e4abd06459cc31.png with policy now +2023-07-12 10:42:31,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_5f9749e51f806c3a7042.png with policy now +2023-07-12 10:42:31,051 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,051 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_b1d38d2764a5721d15ac.png with policy now +2023-07-12 10:42:31,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:42:31,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2550_7e913201741ac8370fe9.png with policy now +2023-07-12 10:42:31,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:42:31,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:42:31,101 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:42:31,103 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:42:31,378 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/di9k3b4f-media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 10:42:31,399 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dv4bydl3-media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 10:42:31,404 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w30qts4a-media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 10:42:31,412 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w22ha3ka-media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 10:42:32,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:37,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:38,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:42:38,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:42:38,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 10:42:38,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 10:42:38,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 10:42:38,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 10:42:38,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:42:39,011 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/upxvzl7t-media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 10:42:39,019 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/glfaly8p-media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 10:42:39,039 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rftl9y8f-media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 10:42:39,041 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ul283g29-media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 10:42:39,964 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:42:42,045 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:42:43,046 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:43,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:43,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:43,790 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:48,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:49,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:42:51,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:42:51,618 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:42:51,618 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:42:51,620 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:42:54,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:42:58,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:42:58,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:42:58,792 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:42:59,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:00,695 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:00,696 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:01,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:01,384 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:01,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:01,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:05,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:09,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:43:10,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:11,537 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:11,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:11,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:11,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:11,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:11,717 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:13,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:43:13,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:43:13,964 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:43:16,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:21,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:21,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:21,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:21,757 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:21,759 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:22,734 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:26,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:29,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:43:29,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:43:29,040 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:43:32,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:32,135 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:32,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:32,136 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:32,138 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:33,751 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:33,751 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:37,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:39,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:43:42,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:42,233 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:42,233 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:42,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:43,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:44,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:43:44,419 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:43:44,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:43:44,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:44,987 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:48,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:52,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:43:52,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:43:52,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:43:52,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:43:53,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:43:56,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:43:56,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:43:59,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:00,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:00,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:00,176 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:02,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:44:02,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:44:02,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:44:02,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:44:04,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:07,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:44:07,025 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:44:08,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:44:08,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:44:08,074 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:44:08,075 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:44:09,968 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:44:09,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:14,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:15,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:15,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:15,175 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:18,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:44:18,041 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:44:20,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:25,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:30,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:30,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:30,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:30,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:35,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:39,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:44:40,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:45,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:44:45,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:44:45,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:44:46,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:51,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:44:56,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:00,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:00,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:00,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:02,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:02,692 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,692 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_35fe8e67bdbd375bad51.png with policy now +2023-07-12 10:45:02,701 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,702 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_e4060fd5427ae0ecee26.png with policy now +2023-07-12 10:45:02,711 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,711 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_8ef00575ef619617eb8f.png with policy now +2023-07-12 10:45:02,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:45:02,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2560_8c7622cbbe1b8f2266c2.png with policy now +2023-07-12 10:45:02,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:02,763 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:02,764 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:02,769 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:03,004 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tn65ea73-media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 10:45:03,015 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e4deabl7-media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 10:45:03,032 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ekhjs1ru-media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 10:45:03,110 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s8rmp1uv-media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 10:45:07,388 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:09,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:45:12,602 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:12,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:13,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:13,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:13,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 10:45:13,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 10:45:13,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 10:45:13,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 10:45:13,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:45:13,696 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1zuourkf-media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 10:45:13,710 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8jbdzebp-media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 10:45:13,710 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xbrod0rb-media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 10:45:13,711 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0a98pobp-media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 10:45:15,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:15,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:15,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:18,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:22,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:22,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:22,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:22,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:23,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:24,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:24,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:28,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:30,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:30,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:30,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:31,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:31,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:31,923 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:31,924 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:33,925 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:35,379 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:35,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:39,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:39,971 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:45:41,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:41,840 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:41,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:41,842 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:44,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:46,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:45:46,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:45:46,214 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:45:46,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:46,397 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:45:50,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:51,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:45:51,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:45:51,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:45:51,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:45:55,560 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:45:57,412 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:45:57,413 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:00,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:01,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:01,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:01,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:02,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:02,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:02,179 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:02,181 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:05,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:08,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:08,430 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:09,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:46:11,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:13,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:13,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:13,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:13,509 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:16,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:16,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:16,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:17,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:19,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:19,660 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:23,527 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:24,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:24,194 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:24,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:24,195 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:29,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:30,677 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:30,678 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:31,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:31,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:31,610 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:34,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:34,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:34,486 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:34,486 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:34,488 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:39,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:46:39,087 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:46:39,088 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:46:39,089 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:46:39,659 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:39,973 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:46:41,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:46:41,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:46:44,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:46,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:46:46,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:46:46,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:46:50,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:46:55,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:00,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:01,610 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:01,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:01,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:06,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:09,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:47:11,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:16,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:16,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:16,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:17,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:22,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:27,857 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:31,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:31,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:31,996 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:33,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:47:36,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 10:47:36,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 10:47:36,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 10:47:36,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 10:47:37,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:47:37,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:38,339 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vrx9tm2v-media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 10:47:40,179 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/80lv8ia6-media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 10:47:41,053 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/briwoxw9-media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 10:47:41,057 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o5w4tttj-media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 10:47:41,150 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:41,150 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_5dcd5f49c5610ed98e57.png with policy now +2023-07-12 10:47:41,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_ceacc8064824cf24b9c3.png with policy now +2023-07-12 10:47:41,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_13fad504fb5c2ac0ee04.png with policy now +2023-07-12 10:47:41,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:47:41,152 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2570_0287eb8f3f80591bc234.png with policy now +2023-07-12 10:47:41,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:47:41,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:47:41,159 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:47:41,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:47:42,106 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7lp5breu-media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 10:47:42,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:47:42,602 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pnrbtnoq-media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 10:47:42,751 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jx8k58q2-media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 10:47:42,785 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b3pzveww-media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 10:47:46,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:47,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:47:47,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:47:47,001 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:47:48,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:47:48,015 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:47:51,660 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:47:51,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:47:51,661 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:47:51,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:47:52,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:58,064 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:47:59,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:47:59,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:02,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:02,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:02,089 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:02,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:02,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:02,154 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:03,155 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:08,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:09,976 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:48:10,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:10,043 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:12,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:12,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:12,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:12,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:14,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:17,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:17,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:17,733 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:20,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:21,060 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:21,061 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:23,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:23,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:23,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:23,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:25,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:30,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:32,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:32,078 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:32,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:32,811 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:32,811 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:33,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:33,376 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:33,376 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:33,378 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:36,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:39,977 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:48:41,273 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:43,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:43,331 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:43,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:43,737 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:43,738 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:43,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:46,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:47,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:48:47,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:48:47,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:48:51,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:48:53,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:48:53,548 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:48:53,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:48:53,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:48:54,347 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:48:54,348 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:48:57,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:02,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:03,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:03,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:03,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:04,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:49:04,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:49:04,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:49:04,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:49:05,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:49:05,366 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:49:08,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:09,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:49:09,258 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:49:09,258 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:49:09,260 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:49:09,978 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:49:13,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:16,383 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:49:16,384 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:49:18,548 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:18,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:18,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:19,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:24,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:29,588 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:33,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:33,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:33,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:34,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:39,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:39,979 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:49:44,981 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:48,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:49:48,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:49:48,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:49:50,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:49:55,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:00,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:03,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:03,549 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:03,550 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:03,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,647 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_784615f77a2b4e7047a9.png with policy now +2023-07-12 10:50:03,648 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,648 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_38ab10654e678fd3808d.png with policy now +2023-07-12 10:50:03,649 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_0362f6603354bb565423.png with policy now +2023-07-12 10:50:03,650 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:50:03,650 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2580_12a798eb5ac3d54ffad6.png with policy now +2023-07-12 10:50:03,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:03,676 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:03,677 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:03,679 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:04,045 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dbupzzwk-media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 10:50:04,047 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9mkqpys6-media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 10:50:04,054 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/djqjry0m-media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 10:50:04,085 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bkz80ns6-media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 10:50:05,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:09,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:50:10,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:11,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:11,708 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 10:50:11,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 10:50:11,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:50:12,105 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uja8oyud-media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 10:50:12,111 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/osh04iah-media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 10:50:12,113 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dtfi4ck0-media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 10:50:12,134 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hgjn5niz-media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 10:50:13,083 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:16,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:19,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:19,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:19,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:22,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:22,595 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:22,596 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:22,596 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:22,598 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:22,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:22,728 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:28,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:32,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:32,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:32,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:32,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:33,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:33,746 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:34,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:34,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:34,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:34,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:39,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:39,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:50:42,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:42,532 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:42,533 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:42,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:44,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:44,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:44,764 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:50:49,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:50:49,690 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:50:49,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:50:49,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:52,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:50:52,656 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:50:52,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:50:52,658 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:50:55,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:50:55,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:50:55,781 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:00,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:03,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:03,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:03,140 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:03,142 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:05,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:05,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:05,178 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:05,741 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:06,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:06,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:09,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:51:11,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:12,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:12,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:12,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:12,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:16,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:18,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:18,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:20,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:20,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:20,287 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:21,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:23,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:23,553 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:23,553 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:23,555 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:27,337 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:29,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:29,054 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:32,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:34,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:34,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:34,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:34,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:35,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:35,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:35,699 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:38,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:38,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:51:38,457 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:51:38,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:51:38,459 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:51:39,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:51:40,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:51:40,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:51:43,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:48,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:51:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:51:50,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:51:54,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:51:59,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:04,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:05,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:05,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:05,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:09,964 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:09,985 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:52:14,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:19,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:20,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:20,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:20,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:25,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:30,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:32,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,788 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_62ad7e090c150c4642cd.png with policy now +2023-07-12 10:52:32,796 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,796 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_2342de4c23f89a71e70a.png with policy now +2023-07-12 10:52:32,806 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_12a6034cf8cfc23b5df1.png with policy now +2023-07-12 10:52:32,816 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:52:32,816 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2590_59ef69c8ce6d4ab2107c.png with policy now +2023-07-12 10:52:32,855 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:52:32,856 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:52:32,856 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:52:32,858 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:52:33,161 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lhct4zfo-media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 10:52:33,174 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/er358v07-media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 10:52:33,174 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m5gy7hv6-media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 10:52:33,189 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1wqs60gw-media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 10:52:35,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:52:35,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 10:52:35,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 10:52:35,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 10:52:35,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 10:52:35,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:52:35,783 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4fgnx7l8-media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 10:52:35,806 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ty3d3z8u-media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 10:52:35,808 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/re2yimyp-media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 10:52:35,814 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/crft1tvd-media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 10:52:35,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:35,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:35,906 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:35,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:39,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:52:41,893 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:43,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:52:46,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:52:47,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:52:50,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:52:50,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:52:52,853 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:52:52,854 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:52:52,855 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:52:52,855 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:52:52,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:52:57,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:52:57,426 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:52:58,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:02,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:02,246 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:02,247 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:02,248 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:04,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:06,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:06,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:06,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:08,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:08,439 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:09,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:09,988 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:53:12,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:12,500 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:12,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:12,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:14,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:19,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:19,456 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:19,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:21,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:21,920 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:21,921 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:22,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:22,487 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:22,487 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:22,489 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:24,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:30,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:30,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:30,473 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:32,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:32,653 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:32,653 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:32,655 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:35,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:37,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:37,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:37,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:39,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:53:40,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:41,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:41,723 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:42,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:42,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:42,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:42,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:45,691 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:50,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:53:52,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:53:52,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:53:52,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:53:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:53:52,741 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:53:54,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:53:54,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:53:54,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:53:54,491 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:53:56,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:02,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:03,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:54:03,758 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:54:04,586 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:54:04,588 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:54:04,589 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:54:04,591 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:54:07,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:07,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:07,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:54:07,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:09,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:54:09,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:54:09,468 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:54:09,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:54:09,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:54:13,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:14,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:54:14,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:54:18,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:22,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:22,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:54:22,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:23,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:28,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:33,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:37,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:37,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:54:37,379 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:38,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:39,992 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:54:43,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:48,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:52,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:54:52,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:54:54,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:54:59,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:03,900 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_404750014d2070b4b00e.png with policy now +2023-07-12 10:55:03,909 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_e5c153d3981bac3820e6.png with policy now +2023-07-12 10:55:03,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_305e21c628b26bb7d55e.png with policy now +2023-07-12 10:55:03,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:55:03,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2600_5996b8fa55b566dccbed.png with policy now +2023-07-12 10:55:03,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:03,977 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:03,977 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:03,981 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:04,228 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mu0v1g0i-media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 10:55:04,243 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4mr4b4hj-media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 10:55:04,255 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q1bneioz-media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 10:55:04,264 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c5sglp9n-media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 10:55:04,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:07,378 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:07,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:09,993 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:55:09,996 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:10,088 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:10,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:10,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 10:55:10,089 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 10:55:10,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 10:55:10,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 10:55:10,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:55:10,473 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ukwhcbq-media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 10:55:10,474 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xhm1fc6m-media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 10:55:10,489 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbd2r1z1-media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 10:55:10,497 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/luk7igg9-media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 10:55:13,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:15,677 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:20,773 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:21,107 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:55:22,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:22,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:23,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:23,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:23,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:23,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:25,809 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:31,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:32,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:32,128 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:32,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:32,590 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:32,591 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:32,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:37,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:37,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:55:37,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:37,908 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:39,994 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:55:42,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:42,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:42,813 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:42,813 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:42,815 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:43,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:43,147 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:48,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:52,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:55:52,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:55:52,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:55:53,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:55:53,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:55:53,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:55:53,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:55:53,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:55:54,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:55:54,163 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:55:58,777 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:03,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:03,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:03,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:03,542 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:04,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:05,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:05,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:07,912 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:07,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:07,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:09,997 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:56:09,999 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:13,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:13,611 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:13,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:13,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:15,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:16,432 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:16,433 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:21,317 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:22,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:22,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:22,913 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:24,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:24,670 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:24,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:24,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:26,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:32,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:34,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:34,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:34,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:34,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:37,362 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:37,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:37,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:37,914 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:38,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:38,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:39,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:56:39,525 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:56:39,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:56:39,527 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:56:39,998 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:56:43,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:48,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:49,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:56:49,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:56:53,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:56:53,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:56:53,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:56:53,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:56:58,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:03,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:08,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:08,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:08,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:08,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:09,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:57:14,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:19,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:23,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:23,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:23,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:24,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:29,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:33,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,799 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_a3c7e4b8847949aa758f.png with policy now +2023-07-12 10:57:33,808 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_d8ef17b50203820bbbe3.png with policy now +2023-07-12 10:57:33,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_5b8b7b248eeeadd01dc0.png with policy now +2023-07-12 10:57:33,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 10:57:33,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2610_3c27c596a729886e6078.png with policy now +2023-07-12 10:57:33,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:57:33,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:57:33,871 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:57:33,873 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:57:34,141 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2yrlbyhu-media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 10:57:34,144 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3e1zpirm-media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 10:57:34,144 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vmzmq9m4-media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 10:57:34,163 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wigy0pwg-media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 10:57:34,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:38,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:38,702 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:38,702 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:40,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:57:40,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:43,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:57:44,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:57:44,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:57:44,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 10:57:44,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 10:57:44,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 10:57:44,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 10:57:44,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 10:57:45,184 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dnfo82zr-media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 10:57:45,196 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/avg62cxt-media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 10:57:45,224 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h3tq612a-media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 10:57:45,231 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nmp4mlm6-media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 10:57:45,944 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:51,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:57:52,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:57:52,942 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:57:52,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:57:52,943 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:57:55,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:57:55,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:57:55,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:57:55,807 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:57:55,808 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:57:56,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:02,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:03,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:03,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:03,766 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:03,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:06,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:06,824 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:07,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:10,002 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:58:10,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:10,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:10,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:13,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:14,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:14,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:14,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:14,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:17,841 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:17,842 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:19,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:24,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:24,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:24,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:24,967 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:24,969 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:25,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:25,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:25,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:28,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:28,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:30,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:34,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:34,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:34,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:34,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:35,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:39,881 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:39,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:40,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:58:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:40,188 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:41,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:44,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:44,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:44,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:44,803 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:46,804 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:51,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:58:51,142 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:58:52,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:58:55,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:58:55,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:58:55,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:58:56,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:58:56,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:58:56,101 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:58:56,102 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:58:58,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:02,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:59:02,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:59:03,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:06,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:59:06,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:59:06,424 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:59:06,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:59:08,814 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:10,005 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:59:10,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:10,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:10,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:59:10,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 10:59:10,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 10:59:10,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 10:59:10,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 10:59:13,178 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 10:59:13,179 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 10:59:14,556 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:19,557 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:24,558 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:25,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:25,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:25,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:59:29,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:34,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:39,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:40,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 10:59:40,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:40,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:40,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 10:59:44,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:49,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:54,761 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 10:59:55,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 10:59:55,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 10:59:55,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:00,719 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:05,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_8d036206f070328af8f6.png with policy now +2023-07-12 11:00:05,104 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,104 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_b1cd6dc23fe6b08880be.png with policy now +2023-07-12 11:00:05,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,114 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_792f17b1a307463180d2.png with policy now +2023-07-12 11:00:05,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:00:05,124 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2620_87526415e87acfa56b82.png with policy now +2023-07-12 11:00:05,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:05,166 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:05,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:05,170 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:05,468 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bk2ur7nb-media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 11:00:05,490 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhk4va1v-media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 11:00:05,507 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l4825rsf-media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 11:00:05,533 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bbrjrit0-media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 11:00:06,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:08,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:08,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 11:00:08,488 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 11:00:08,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 11:00:08,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 11:00:08,490 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:00:08,819 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wj7r3fds-media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 11:00:08,849 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v8te0cvf-media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 11:00:08,852 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j5r73jxf-media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 11:00:08,853 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gbtyx5vj-media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 11:00:10,007 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:00:10,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:10,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:10,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:11,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:14,619 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:16,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:19,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:22,918 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:24,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:24,412 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:24,413 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:24,415 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:25,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:25,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:25,654 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:28,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:30,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:30,527 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:33,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:34,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:34,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:34,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:34,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:38,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:40,009 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:00:41,547 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:41,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:41,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:41,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:41,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:00:44,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:45,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:45,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:45,146 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:45,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:50,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:52,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:00:52,565 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:00:55,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:00:55,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:00:55,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:00:55,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:00:55,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:00:57,781 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:00:57,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:00:57,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:01,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:03,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:03,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:06,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:06,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:06,477 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:06,478 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:06,480 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:10,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:01:11,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:13,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:13,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:13,382 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:14,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:14,819 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:16,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:16,545 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:16,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:16,547 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:17,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:22,700 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:25,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:25,843 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:27,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:27,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:27,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:27,243 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:28,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:29,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:29,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:29,117 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:33,310 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:36,863 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:36,864 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:37,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:37,173 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:37,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:37,176 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:39,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:40,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:01:41,885 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:01:41,886 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:01:41,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:01:41,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:01:44,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:44,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:44,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:01:44,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:47,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:01:47,880 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:01:49,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:54,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:01:59,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:01:59,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:01:59,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:00,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:05,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:10,011 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:02:11,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:14,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:14,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:14,205 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:16,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:21,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:26,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:29,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:29,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:29,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:32,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:36,268 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_af826553a0f3457c21a4.png with policy now +2023-07-12 11:02:36,277 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,278 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_1e363e4fe52a8dd1f3d5.png with policy now +2023-07-12 11:02:36,287 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,287 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_01eb7980170d3a100648.png with policy now +2023-07-12 11:02:36,298 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:02:36,298 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2630_f752872a939e31b6a407.png with policy now +2023-07-12 11:02:36,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:02:36,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:02:36,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:02:36,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:02:36,579 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/c21mto1z-media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 11:02:36,589 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l41mzesa-media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 11:02:36,604 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/67uslcst-media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 11:02:36,605 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/icjluwj2-media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 11:02:37,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:40,012 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:02:42,469 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:43,211 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:02:43,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:02:43,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 11:02:43,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 11:02:43,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 11:02:43,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 11:02:43,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:02:43,572 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/87sll2x1-media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 11:02:43,589 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/swopib3z-media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 11:02:43,594 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u2aev0gy-media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 11:02:43,603 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6pynq7oc-media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 11:02:44,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:44,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:44,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:46,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:02:48,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:53,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:02:54,230 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:02:55,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:02:55,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:02:55,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:02:55,073 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:02:59,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:02:59,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:02:59,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:02:59,288 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:04,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:05,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:05,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:05,224 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:05,226 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:05,247 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:09,799 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:10,014 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:03:14,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:14,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:14,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:03:15,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:15,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:15,476 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:15,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:15,478 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:16,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:16,265 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:20,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:25,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:27,128 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:27,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:27,130 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:27,132 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:27,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:27,283 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:29,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:29,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:29,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:03:31,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:36,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:37,491 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:37,492 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:37,492 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:37,494 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:38,299 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:38,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:40,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:03:42,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:44,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:03:47,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:47,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:47,443 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:47,443 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:47,445 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:49,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:03:49,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:03:53,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:56,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:03:56,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:03:56,999 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:03:57,001 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:03:59,002 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:03:59,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:03:59,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:03:59,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:00,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:04:00,568 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:04:04,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:08,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:04:08,260 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:04:08,261 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:04:08,263 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:04:10,017 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:04:10,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:11,583 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:04:11,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:04:12,829 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:04:12,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:04:12,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:04:12,833 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:04:14,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:14,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:14,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:15,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:20,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:22,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:04:22,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:04:25,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:29,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:29,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:29,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:31,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:36,959 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:40,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:04:42,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:44,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:44,911 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:44,911 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:04:47,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:52,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:57,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:04:59,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:04:59,951 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:04:59,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:02,987 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:07,460 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_cbe3a2e2923fea324bbe.png with policy now +2023-07-12 11:05:07,502 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,502 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_ece7f7211bafcdaa62b2.png with policy now +2023-07-12 11:05:07,503 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,504 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_8852be70b2d4d0b9cda6.png with policy now +2023-07-12 11:05:07,507 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:05:07,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2640_60a5b23b74be955f2c27.png with policy now +2023-07-12 11:05:07,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:07,539 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:07,539 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:07,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:07,762 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/69hiaa3o-media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 11:05:07,814 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r74jo1n6-media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 11:05:07,816 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0d5nrzrh-media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 11:05:07,818 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w2e4awfw-media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 11:05:08,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:10,019 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:05:14,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:15,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:05:15,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:05:15,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:17,972 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:17,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:17,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 11:05:17,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 11:05:17,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 11:05:17,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 11:05:17,974 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:05:18,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:19,751 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k8wj7adu-media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 11:05:20,026 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rpfzb5h5-media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 11:05:20,054 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vvgjajin-media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 11:05:20,273 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fjmmeyda-media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 11:05:20,375 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:25,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:27,779 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:27,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:27,781 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:27,783 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:28,989 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:28,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:30,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:05:30,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:05:30,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:31,425 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:36,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:37,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:37,915 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:37,916 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:37,918 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:40,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:40,004 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:40,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:05:42,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:45,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:05:45,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:05:45,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:05:47,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:47,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:47,871 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:47,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:47,874 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:05:51,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:05:51,022 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:05:52,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:58,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:05:58,601 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:05:58,603 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:05:58,603 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:05:58,605 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:00,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:00,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:00,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:02,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:02,039 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:03,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:08,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:08,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:08,437 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:08,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:08,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:10,021 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:06:13,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:13,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:14,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:15,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:15,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:15,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:18,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:18,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:18,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:18,358 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:20,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:24,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:24,305 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:26,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:29,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:29,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:29,034 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:29,036 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:31,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:31,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:31,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:31,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:35,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:35,323 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:37,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:38,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:38,825 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:38,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:38,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:40,022 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:06:43,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:43,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:06:43,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:06:43,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:06:43,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:06:46,023 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:06:46,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:06:46,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:06:46,340 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:06:46,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:06:48,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:53,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:06:58,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:01,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:01,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:01,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:04,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:09,068 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:10,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:07:15,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:16,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:16,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:16,024 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:20,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:25,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:30,097 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:31,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:31,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:31,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:35,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:37,273 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,273 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_e1afc2c90dd83c53afe8.png with policy now +2023-07-12 11:07:37,289 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,289 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_3a9f356aa83d5b8c3154.png with policy now +2023-07-12 11:07:37,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,300 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_feb61af945632b2e5a0d.png with policy now +2023-07-12 11:07:37,632 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:07:37,632 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2650_bd16c2dda6cb242546bd.png with policy now +2023-07-12 11:07:37,649 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/p7otx2m5-media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 11:07:37,651 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mxm33j5k-media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 11:07:37,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:07:37,675 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:07:37,675 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:07:37,677 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:07:37,754 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8j82gxbp-media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 11:07:38,009 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdamez17-media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 11:07:40,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:07:40,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:41,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:07:41,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:07:41,648 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 11:07:41,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 11:07:41,649 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 11:07:41,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 11:07:41,650 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:07:42,026 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gjiwh2r6-media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 11:07:42,031 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sfwldm3q-media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 11:07:42,044 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8fse9ha6-media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 11:07:42,066 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hynv9ieb-media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 11:07:46,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:07:46,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:07:46,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:07:46,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:47,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:07:51,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:52,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:07:56,831 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:07:58,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:07:58,455 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:07:58,456 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:07:58,458 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:01,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:01,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:01,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:02,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:03,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:03,683 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:07,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:07,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:07,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:07,886 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:07,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:10,024 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:08:12,790 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:14,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:14,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:16,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:16,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:16,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:18,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:18,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:18,278 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:18,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:18,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:24,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:25,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:25,718 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:27,952 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:27,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:27,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:27,957 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:29,958 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:31,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:31,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:31,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:35,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:36,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:36,736 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:38,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:38,580 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:38,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:38,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:40,025 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:08:41,026 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:46,118 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:08:46,119 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:08:46,119 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:08:46,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:47,990 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:47,991 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:48,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:48,987 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:48,987 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:48,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:08:51,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:57,681 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:08:59,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:08:59,008 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:08:59,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:08:59,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:08:59,103 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:08:59,105 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:09:01,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:01,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:01,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:02,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:08,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:09,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:09:09,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:09:09,579 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:09:09,581 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:09:10,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:09:10,036 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:09:10,040 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:09:14,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:14,531 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:09:14,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:09:14,534 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:09:14,535 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:09:16,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:16,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:16,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:20,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:21,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:09:21,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:09:25,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:30,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:31,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:31,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:31,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:35,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:40,032 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:09:41,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:46,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:46,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:09:46,365 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:09:46,365 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:09:51,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:09:56,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:01,422 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:01,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:01,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:01,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:06,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:08,859 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,859 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_39f7775ce6273f5abdd3.png with policy now +2023-07-12 11:10:08,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_be22a91ba76075904920.png with policy now +2023-07-12 11:10:08,878 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,879 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_570101fe2f3838813987.png with policy now +2023-07-12 11:10:08,889 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:10:08,889 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2660_049933627fe17227a897.png with policy now +2023-07-12 11:10:08,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:08,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:08,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:08,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:09,187 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/b3vkw3xu-media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 11:10:09,218 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0vqmt49s-media/images/validation_2660_049933627fe17227a897.png +2023-07-12 11:10:09,224 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f0bi2h2k-media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 11:10:09,244 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9e87psg-media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 11:10:10,033 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:10:12,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:16,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:10:16,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:16,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_049933627fe17227a897.png +2023-07-12 11:10:16,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 11:10:16,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 11:10:16,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 11:10:16,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:10:16,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:16,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:16,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:16,751 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2pqxqh68-media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 11:10:16,786 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wg92egh7-media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 11:10:16,792 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/52xtaraa-media/images/validation_2660_049933627fe17227a897.png +2023-07-12 11:10:16,840 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yjx44ah0-media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 11:10:17,852 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:19,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:23,714 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:27,380 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:28,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:28,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:28,795 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:28,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:28,798 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:31,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:31,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:31,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:34,643 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:38,398 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:10:38,399 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:39,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:39,494 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:39,495 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:39,498 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:40,034 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:10:40,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:45,347 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:46,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:10:46,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:10:46,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:10:49,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:10:49,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:10:49,118 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:10:49,121 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:10:49,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:10:49,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:10:51,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:10:56,346 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:00,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:00,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:00,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:00,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:00,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:00,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:02,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:02,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:02,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:02,417 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:07,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:10,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:11:10,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:10,872 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:10,872 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:10,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:11,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:11,455 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:13,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:17,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:17,838 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:17,839 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:18,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:20,620 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:20,621 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:20,622 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:20,624 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:22,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:22,706 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:24,180 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:30,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:30,708 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:30,710 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:30,710 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:30,712 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:33,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:33,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:33,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:33,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:33,725 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:35,584 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:40,036 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:11:41,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:41,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:41,375 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:41,375 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:41,377 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:44,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:44,743 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:46,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:46,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:11:46,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:11:46,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:11:46,214 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:11:48,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:11:48,022 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:11:48,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:11:51,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:11:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:11:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:11:56,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:01,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:03,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:03,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:03,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:06,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:10,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:12:12,038 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:17,039 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:18,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:18,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:18,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:22,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:27,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:32,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:33,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:33,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:33,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:37,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:40,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:12:40,562 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,562 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_b7c6a59e577b6aff9423.png with policy now +2023-07-12 11:12:40,572 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,572 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_8f7a128a9aebab91d68e.png with policy now +2023-07-12 11:12:40,582 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_abd6457914db5ecb11f5.png with policy now +2023-07-12 11:12:40,593 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:12:40,593 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2670_81b157e520543297899b.png with policy now +2023-07-12 11:12:40,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:12:40,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:12:40,634 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:12:40,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:12:40,907 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ky8wo4qf-media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 11:12:40,928 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xdq4m6zn-media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 11:12:40,940 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kgj989gd-media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 11:12:40,963 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z3yyocan-media/images/validation_2670_81b157e520543297899b.png +2023-07-12 11:12:42,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:47,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:48,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:12:48,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:12:48,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:12:50,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 11:12:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 11:12:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 11:12:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_81b157e520543297899b.png +2023-07-12 11:12:51,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:12:51,492 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8u2pi9kz-media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 11:12:51,524 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/09l1owfd-media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 11:12:51,553 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0dwyzo1e-media/images/validation_2670_81b157e520543297899b.png +2023-07-12 11:12:51,566 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmb5321e-media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 11:12:53,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:59,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:12:59,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:00,000 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:00,000 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:00,003 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:02,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:02,102 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:03,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:03,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:03,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:05,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:10,039 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:13:10,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:10,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:10,362 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:10,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:10,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:13,119 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:13,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:15,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:18,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:18,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:18,139 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:20,680 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:20,969 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:20,970 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:20,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:20,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:24,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:24,143 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:26,532 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:31,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:31,506 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:31,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:31,508 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:32,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:33,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:33,713 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:33,713 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:35,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:35,160 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:37,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:40,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:13:41,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:41,892 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:41,893 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:41,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:42,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:46,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:46,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:48,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:48,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:13:48,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:13:48,821 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:13:52,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:13:52,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:13:52,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:13:52,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:13:53,402 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:13:57,416 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:13:57,417 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:13:59,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:02,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:14:02,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:14:02,407 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:14:02,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:14:04,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:04,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:04,202 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:04,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:08,434 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:14:08,435 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:14:09,517 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:10,041 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:14:12,683 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:14:12,684 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:14:12,685 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:14:12,687 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:14:14,689 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:17,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:14:17,265 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:14:17,266 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:14:17,268 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:14:19,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:19,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:19,203 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:19,452 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:14:19,453 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:14:20,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:25,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:30,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:34,202 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:34,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:34,245 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:35,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:40,042 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:14:41,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:46,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:49,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:14:49,304 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:14:49,305 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:14:51,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:14:56,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:01,372 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:04,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:15:04,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:04,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:06,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:10,043 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:15:11,662 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,663 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_3ca3ef00974e8561788d.png with policy now +2023-07-12 11:15:11,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:11,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_7cb40a3f835111c25bc8.png with policy now +2023-07-12 11:15:11,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_1128a7fc997078f6857d.png with policy now +2023-07-12 11:15:11,691 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:15:11,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2680_5968f013df449c836103.png with policy now +2023-07-12 11:15:11,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:11,731 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:11,731 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:11,733 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:11,989 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ctwth8si-media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 11:15:12,015 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xpivlpww-media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 11:15:12,039 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3bqdhnsa-media/images/validation_2680_5968f013df449c836103.png +2023-07-12 11:15:12,040 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rrb9xotb-media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 11:15:14,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:14,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 11:15:14,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_5968f013df449c836103.png +2023-07-12 11:15:14,778 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 11:15:14,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 11:15:14,779 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:15:15,191 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k144eapa-media/images/validation_2680_5968f013df449c836103.png +2023-07-12 11:15:15,192 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2uo5zdfv-media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 11:15:15,197 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/toik5y3j-media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 11:15:15,208 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2fz9uxol-media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 11:15:17,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:19,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:15:19,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:19,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:22,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:22,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:25,794 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:27,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:31,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:31,821 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:31,821 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:31,823 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:32,824 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:34,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:34,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:15:34,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:36,813 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:36,814 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:38,703 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:40,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:15:41,726 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:41,728 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:41,728 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:41,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:43,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:47,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:47,832 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:48,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:49,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:15:49,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:15:51,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:15:51,620 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:15:51,621 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:15:51,623 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:15:54,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:15:58,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:15:58,849 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:15:59,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:01,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:01,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:01,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:01,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:04,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:04,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:04,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:09,865 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:09,866 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:10,045 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:16:10,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:11,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:11,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:11,899 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:11,901 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:15,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:19,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:19,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:16:19,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:20,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:21,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:21,135 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:22,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:22,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:22,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:22,107 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:26,661 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:31,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:32,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:32,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:32,612 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:32,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:32,614 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:32,615 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:34,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:16:34,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:34,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:37,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:40,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:16:42,524 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:42,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:42,526 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:42,526 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:42,528 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:43,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:43,175 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:47,233 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:16:47,234 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:16:47,235 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:16:47,236 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:16:47,699 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:16:49,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:16:49,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:16:53,613 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:16:54,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:16:54,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:16:58,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:03,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:04,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:04,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:04,573 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:08,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:10,047 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:17:14,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:19,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:19,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:19,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:19,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:24,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:29,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:34,694 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:34,695 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:34,695 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:35,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:40,048 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:17:41,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:41,809 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,809 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_f6facb6c2b52824b8087.png with policy now +2023-07-12 11:17:41,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_ea076148ce2508581f4a.png with policy now +2023-07-12 11:17:41,828 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,828 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_6b92b33ac5d9538cd14f.png with policy now +2023-07-12 11:17:41,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:17:41,838 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2690_09839606b108aef4ba45.png with policy now +2023-07-12 11:17:41,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:17:41,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:17:41,878 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:17:41,880 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:17:42,147 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/311w19x7-media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 11:17:42,157 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h6vdgr2h-media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 11:17:42,173 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jiomayv6-media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 11:17:42,211 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/45r42ksx-media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 11:17:46,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:49,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:17:49,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:17:49,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 11:17:49,509 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 11:17:49,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 11:17:49,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 11:17:49,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:17:49,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:17:49,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:17:49,697 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:17:50,055 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/doy6em4i-media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 11:17:50,059 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1fzuu2ea-media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 11:17:50,069 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rilbp60s-media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 11:17:50,074 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5337gwsy-media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 11:17:52,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:17:52,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:17:57,707 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:00,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:01,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:01,201 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:01,201 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:01,203 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:03,204 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:05,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:05,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:05,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:08,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:10,049 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:18:10,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:10,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:10,172 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:10,174 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:11,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:11,548 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:14,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:19,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:20,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:20,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:20,973 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:20,974 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:20,975 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:20,977 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:22,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:22,569 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:24,967 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:30,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:31,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:31,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:31,768 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:31,770 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:33,586 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:33,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:35,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:35,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:35,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:35,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:40,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:18:41,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:42,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:42,756 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:42,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:42,758 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:44,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:44,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:46,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:50,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:18:50,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:18:50,410 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:18:52,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:18:52,830 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:18:52,831 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:18:52,832 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:18:52,834 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:18:55,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:18:55,856 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:18:57,649 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:02,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:19:02,848 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:02,848 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:19:02,848 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:19:02,851 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:19:05,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:05,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:05,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:06,872 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:19:06,873 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:19:08,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:10,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:19:13,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:19:13,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:13,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:19:13,191 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:19:13,193 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:19:17,834 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:19:17,835 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:19:17,835 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:19:17,837 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:19:17,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:19:17,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:19:18,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:20,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:20,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:20,545 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:23,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:28,617 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:28,908 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:19:33,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:35,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:35,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:35,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:39,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:40,053 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:19:45,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:50,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:19:50,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:19:50,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:19:50,693 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:19:55,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:00,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:05,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:05,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:05,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:05,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:10,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:20:11,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:12,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_2a9c6e3e9705c484c43d.png with policy now +2023-07-12 11:20:12,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,399 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_951a6312bd4ff445e3bf.png with policy now +2023-07-12 11:20:12,409 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,409 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_cd74dc827e683d49ff49.png with policy now +2023-07-12 11:20:12,420 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:20:12,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2700_9d44392c5a9c6cbb63a6.png with policy now +2023-07-12 11:20:12,467 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:12,469 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:12,469 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:12,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:12,725 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ijw2ulkw-media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 11:20:12,727 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a8p6o5sk-media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 11:20:12,748 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o21w7tqt-media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 11:20:12,748 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mz84i902-media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 11:20:13,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:13,232 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 11:20:13,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 11:20:13,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 11:20:13,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 11:20:13,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:20:13,572 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u6tus951-media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 11:20:13,576 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ospjqimt-media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 11:20:13,579 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5tjc368x-media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 11:20:13,602 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3xm4k94c-media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 11:20:16,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:20,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:20,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:20,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:21,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:22,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:24,251 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:20:27,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:31,753 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:31,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:31,755 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:31,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:32,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:35,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:35,270 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:20:35,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:35,896 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:35,897 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:38,453 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:40,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:20:41,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:41,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:41,689 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:41,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:43,692 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:46,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:46,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:20:49,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:51,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:20:51,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:20:51,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:20:52,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:20:52,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:20:52,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:20:52,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:20:54,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:20:57,307 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:20:57,308 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:00,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:02,021 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:02,023 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:02,023 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:02,025 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:05,625 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:06,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:06,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:06,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:08,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:08,326 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:10,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:21:11,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:12,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:12,254 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:12,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:12,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:16,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:19,584 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:19,585 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:21,698 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:21,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:21,802 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:21,803 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:22,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:22,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:22,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:22,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:26,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:30,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:30,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:32,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:32,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:32,577 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:32,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:32,579 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:37,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:37,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:37,421 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:37,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:40,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:21:41,620 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:41,621 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:43,784 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:43,916 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:43,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:43,918 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:43,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:48,593 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:21:48,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:21:48,594 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:21:48,596 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:21:49,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:52,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:21:52,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:21:52,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:21:52,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:21:52,638 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:21:54,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:21:59,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:04,542 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:07,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:07,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:07,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:09,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:10,059 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:22:15,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:20,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:22,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:22,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:22,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:25,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:30,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:35,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:37,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:37,641 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:37,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:40,060 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:22:41,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:42,652 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,652 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_de3f2b25e35e4537c17e.png with policy now +2023-07-12 11:22:42,661 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,661 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_5ca25be3dc62ed26aa0d.png with policy now +2023-07-12 11:22:42,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,671 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_1328ec76d8379d97a378.png with policy now +2023-07-12 11:22:42,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:22:42,681 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2710_6e2b6aa87197c3eba430.png with policy now +2023-07-12 11:22:42,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:22:42,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:22:42,721 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:22:42,723 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:22:43,002 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cths0upt-media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 11:22:43,011 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gjw1rpsi-media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 11:22:43,019 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xjc0hddw-media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 11:22:43,026 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g6hy87ld-media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 11:22:46,760 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:47,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:22:47,960 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:22:47,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 11:22:47,961 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 11:22:47,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 11:22:47,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 11:22:47,962 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:22:48,341 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9y553guv-media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 11:22:48,356 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mwvvk1wp-media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 11:22:48,366 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qkk3mhyr-media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 11:22:48,367 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nwpbxe56-media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 11:22:52,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:22:52,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:22:52,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:22:52,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:53,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:22:57,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:22:58,977 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:02,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:02,255 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:02,255 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:02,257 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:03,258 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:07,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:07,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:08,652 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:09,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:09,997 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:10,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:23:12,596 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:12,597 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:12,597 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:12,599 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:14,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:19,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:21,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:21,016 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:22,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:22,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:23:22,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:23,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:23,003 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:23,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:23,004 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:25,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:30,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:32,033 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:32,034 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:33,530 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:33,531 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:33,532 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:33,534 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:35,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:37,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:37,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:40,061 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:23:40,789 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:43,051 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:43,052 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:43,750 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:43,752 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:43,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:43,754 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:46,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:51,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:23:52,598 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:23:52,598 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:23:54,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:23:54,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:23:54,030 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:23:54,032 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:23:54,310 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:23:54,311 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:23:56,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:01,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:04,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:24:04,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:24:04,006 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:24:04,008 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:24:05,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:24:05,328 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:24:07,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:08,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:08,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:08,167 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:10,063 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:24:12,277 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:13,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:24:13,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:24:13,799 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:24:13,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:24:16,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:24:16,344 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:24:18,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:18,398 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:24:18,399 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:24:18,399 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:24:18,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:24:23,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:23,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:23,166 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:23,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:27,362 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:24:27,363 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:24:28,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:33,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:38,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:38,196 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:38,197 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:38,241 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:40,064 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:24:44,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:49,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:53,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:24:53,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:24:53,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:24:54,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:24:59,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:04,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:08,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:08,408 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:08,408 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:09,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:10,065 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:25:12,866 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_fb1e12ca3be46bca7950.png with policy now +2023-07-12 11:25:12,875 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,875 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_cab768f5b1c4da57e828.png with policy now +2023-07-12 11:25:12,885 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,885 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_9ebc2d1c275cb06de97e.png with policy now +2023-07-12 11:25:12,897 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:25:12,898 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2720_62fa4b4be6b75191315d.png with policy now +2023-07-12 11:25:12,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:12,938 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:12,939 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:12,941 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:13,199 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gxwf057q-media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 11:25:13,216 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9zqntcs0-media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 11:25:13,229 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3z5q8hwa-media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 11:25:13,287 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dfz4offs-media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 11:25:14,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:20,001 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:22,550 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:22,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:22,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:22,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 11:25:22,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 11:25:22,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 11:25:22,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 11:25:22,703 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:25:23,097 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8i1l33xv-media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 11:25:23,099 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2rmua22y-media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 11:25:23,104 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9ff1qg20-media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 11:25:23,107 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/00z30fnf-media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 11:25:24,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:24,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:24,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:25,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:30,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:32,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:32,535 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:32,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:32,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:33,720 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:33,721 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:36,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:39,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:39,796 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:39,796 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:40,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:25:42,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:42,538 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:42,540 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:42,540 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:42,541 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:44,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:44,742 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:47,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:52,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:25:52,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:25:52,575 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:25:52,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:25:52,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:25:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:25:54,800 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:25:54,801 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:25:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:25:55,760 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:25:57,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:02,775 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:03,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:03,377 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:03,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:03,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:06,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:06,775 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:08,326 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:10,067 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:26:10,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:10,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:10,688 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:26:13,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:13,746 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:13,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:13,747 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:13,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:17,795 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:17,796 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:18,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:24,279 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:24,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:24,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:24,281 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:24,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:26,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:26,210 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:26,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:26:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:29,049 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:29,764 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:34,889 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:34,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:34,891 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:34,891 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:34,893 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:40,067 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:40,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:40,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:26:40,072 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:41,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:41,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:41,806 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:26:44,767 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:44,769 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:44,769 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:44,771 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:45,772 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:49,629 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:26:49,630 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:26:49,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:26:49,631 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:26:51,084 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:26:51,085 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:26:51,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:56,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:26:57,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:26:57,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:26:57,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:02,103 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:27:02,108 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:07,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:10,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:27:12,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:12,037 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:12,037 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:13,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:18,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:23,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:27,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:27,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:27,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:28,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:33,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:38,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:40,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:27:42,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:42,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:42,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:43,198 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:44,284 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,284 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_468e29d1792ddfc31f96.png with policy now +2023-07-12 11:27:44,294 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,294 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_ec678e94be72a14ad671.png with policy now +2023-07-12 11:27:44,304 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,304 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_7a6b051b3cd7a36b4c2e.png with policy now +2023-07-12 11:27:44,314 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:27:44,315 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2730_a1852227cbd29c30acc0.png with policy now +2023-07-12 11:27:44,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:27:44,355 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:27:44,355 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:27:44,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:27:44,640 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/da6qzix1-media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 11:27:44,641 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k16v5f2r-media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 11:27:44,646 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1l1rv8a5-media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 11:27:44,664 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/40jxxe9e-media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 11:27:46,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:27:46,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 11:27:46,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 11:27:46,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 11:27:46,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 11:27:46,420 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:27:46,834 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zotxchec-media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 11:27:46,835 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r4oj4kbu-media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 11:27:46,843 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w96eh0h2-media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 11:27:46,851 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1qayuh72-media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 11:27:48,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:54,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:27:54,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:27:57,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:27:57,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:27:57,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:27:57,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:27:59,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:03,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:03,398 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:03,398 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:03,400 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:05,400 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:08,457 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:08,458 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:10,071 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:28:10,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:12,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:12,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:12,358 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:12,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:12,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:12,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:12,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:16,511 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:19,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:19,476 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:21,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:23,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:23,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:23,136 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:23,138 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:26,783 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:27,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:27,656 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:27,656 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:30,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:30,491 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:31,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:33,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:33,394 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:33,394 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:33,396 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:37,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:40,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:28:41,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:41,508 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:42,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:42,668 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:42,669 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:43,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:43,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:43,236 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:43,236 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:43,238 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:48,922 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:52,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:28:52,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:28:53,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:28:53,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:28:53,321 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:28:53,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:28:54,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:28:57,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:28:57,901 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:28:57,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:28:59,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:03,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:29:03,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:29:03,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:29:03,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:29:03,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:29:03,792 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:29:04,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:10,073 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:29:10,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:13,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:13,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:13,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:13,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:29:13,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:29:13,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:29:13,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:29:14,811 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:29:14,812 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:29:15,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:19,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:29:19,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:29:19,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:29:19,060 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:29:21,720 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:25,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:29:25,829 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:29:26,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:28,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:28,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:28,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:32,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:37,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:40,074 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:29:43,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:43,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:43,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:43,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:48,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:53,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:29:58,543 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:29:58,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:29:58,544 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:29:59,622 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:04,623 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:09,624 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:10,075 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:30:13,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,347 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_a1a975ef1b62e7916638.png with policy now +2023-07-12 11:30:13,356 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,356 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_3a9de177b9a56588f2fd.png with policy now +2023-07-12 11:30:13,367 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,367 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_88328b2a91d7dcdc342e.png with policy now +2023-07-12 11:30:13,380 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:30:13,380 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2740_1eadc0d7d441ac97c181.png with policy now +2023-07-12 11:30:13,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:13,424 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:13,425 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:13,426 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:13,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:13,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:13,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:30:13,799 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vkxysg2j-media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 11:30:13,800 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9i8vxxr-media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 11:30:13,996 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/abtycagn-media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 11:30:14,020 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/86yf18je-media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 11:30:14,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:19,856 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:21,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:30:21,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:21,155 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 11:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 11:30:21,156 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 11:30:21,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 11:30:21,161 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:30:21,518 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rhcz1xl4-media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 11:30:21,521 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/djhrhvbd-media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 11:30:21,558 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2kfg05x8-media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 11:30:21,570 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/86b8jalj-media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 11:30:23,441 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:25,443 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:28,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:28,563 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:28,564 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:30:31,114 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:32,172 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:32,488 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:32,489 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:32,489 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:32,491 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:36,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:40,076 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:30:42,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:42,448 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:42,449 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:42,450 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:42,452 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:44,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:44,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:44,087 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:30:44,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:30:44,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:47,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:52,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:30:52,281 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:30:52,282 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:30:52,283 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:30:53,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:56,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:30:56,214 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:30:58,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:30:59,391 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:30:59,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:30:59,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:02,474 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:02,475 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:02,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:02,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:03,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:08,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:08,233 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:09,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:10,077 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:31:12,868 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:12,869 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:12,869 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:12,870 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:14,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:31:14,734 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:31:14,735 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:14,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:19,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:20,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:20,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:22,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:22,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:22,932 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:22,934 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:25,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:30,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:31:30,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:31:30,096 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:30,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:32,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:32,517 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:33,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:33,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:33,279 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:33,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:35,697 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:40,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:31:41,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:43,416 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:43,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:43,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:43,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:44,535 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:44,536 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:45,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:31:45,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:31:45,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:31:47,296 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:48,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:31:48,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:31:48,523 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:31:48,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:31:53,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:31:56,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:31:56,554 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:31:58,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:00,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:00,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:00,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:03,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:08,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:10,079 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:32:14,080 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:15,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:15,736 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:15,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:19,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:24,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:29,947 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:30,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:35,939 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:40,080 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:32:41,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:42,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:42,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_809fb9a00d33011ef1b7.png with policy now +2023-07-12 11:32:42,984 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:42,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_35d816078fba6fbc5f20.png with policy now +2023-07-12 11:32:42,995 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:42,995 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_e2650e26371a6e674cfa.png with policy now +2023-07-12 11:32:43,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:32:43,006 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2750_44058a9b1a42011478fa.png with policy now +2023-07-12 11:32:43,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:32:43,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:32:43,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:32:43,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:32:43,462 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/id5cdbl9-media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 11:32:43,524 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bdgkr8c9-media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 11:32:43,539 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2wkn3hli-media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 11:32:43,605 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wexi19l1-media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 11:32:44,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:32:44,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 11:32:44,859 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 11:32:44,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 11:32:44,860 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 11:32:44,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:32:45,324 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ky8cuuqs-media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 11:32:45,351 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j0uz754x-media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 11:32:45,398 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/r4xrccpl-media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 11:32:45,404 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/lrzj9u28-media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 11:32:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:32:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:32:45,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:32:46,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:51,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:32:53,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:32:56,877 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:32:57,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:00,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:00,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:00,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:02,921 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:02,923 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:02,923 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:02,924 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:02,926 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:08,487 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:08,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:08,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:10,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:33:12,935 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:12,937 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:12,937 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:12,939 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:13,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:15,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:15,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:15,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:19,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:20,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:20,913 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:23,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:23,503 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:23,503 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:23,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:25,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:30,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:30,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:30,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:30,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:32,931 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:32,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:33,651 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:33,654 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:33,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:33,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:35,890 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:40,083 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:33:41,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:43,403 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:43,405 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:43,405 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:43,408 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:45,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:45,201 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:33:45,883 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:33:45,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:33:45,884 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:33:46,231 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:51,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:54,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:33:54,446 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:33:54,446 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:33:54,448 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:33:56,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:33:57,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:33:57,221 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:01,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:01,438 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:01,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:04,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:34:04,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:34:04,516 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:34:04,518 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:34:07,679 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:09,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:34:09,239 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:10,084 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:34:13,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:14,892 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:34:14,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:34:14,894 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:34:14,896 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:34:16,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:16,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:16,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:18,519 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:19,685 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:34:19,687 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:34:19,687 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:34:19,690 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:34:21,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:34:21,259 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:24,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:29,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:31,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:31,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:31,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:33,281 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:34:35,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:40,035 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:40,085 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:34:45,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:46,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:34:46,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:34:46,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:34:50,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:34:55,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:00,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:01,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:01,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:01,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:06,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:10,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:35:11,087 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:14,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,148 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_70b32648987a29c582d5.png with policy now +2023-07-12 11:35:14,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_aab6a0c7b6bb6712ffaa.png with policy now +2023-07-12 11:35:14,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,167 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png with policy now +2023-07-12 11:35:14,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:35:14,177 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2760_0eccab278b3c49ff65d0.png with policy now +2023-07-12 11:35:14,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:14,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:14,217 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:14,219 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:14,549 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nqm6rm4c-media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 11:35:14,552 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wmrfgjp9-media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 11:35:14,571 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0vuypjzu-media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 11:35:14,574 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/64wgf1lp-media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 11:35:16,220 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:16,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:16,955 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:16,955 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:21,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:21,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:21,593 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:21,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 11:35:21,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 11:35:21,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 11:35:21,595 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 11:35:21,596 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:35:21,924 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kibcfo9b-media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 11:35:21,931 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/00twsuqf-media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 11:35:21,957 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9azkhohz-media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 11:35:21,958 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jaza5ul4-media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 11:35:23,835 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:26,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:31,992 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:32,794 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:32,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:32,795 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:33,364 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:33,366 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:33,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:33,369 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:33,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:33,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:37,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:40,088 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:35:42,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:42,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:42,999 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:42,999 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:43,001 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:45,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:45,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:47,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:35:47,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:35:47,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:35:48,397 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:53,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:35:53,369 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:35:53,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:35:53,371 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:35:54,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:35:57,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:35:57,653 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:35:59,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:02,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:36:02,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:02,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:03,495 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:03,497 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:03,497 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:03,499 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:05,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:09,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:09,670 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:10,089 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:36:11,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:13,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:13,631 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:13,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:13,634 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:16,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:18,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:36:18,350 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:18,351 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:21,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:21,936 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:21,937 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:24,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:24,068 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:24,068 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:24,069 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:27,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:32,587 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:33,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:36:33,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:33,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:33,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:34,319 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:34,320 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:34,321 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:34,322 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:37,866 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:40,090 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:36:42,897 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:44,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:44,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:44,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:44,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:45,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:45,973 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:47,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:48,745 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:36:48,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:36:49,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:36:49,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:36:49,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:36:49,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:36:53,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:36:57,992 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:36:57,993 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:36:58,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:03,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:03,940 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:03,941 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:03,941 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:09,019 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:10,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:37:14,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:19,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:19,034 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:19,035 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:20,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:25,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:30,094 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:34,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:34,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:34,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:35,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:40,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:37:41,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:43,387 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,388 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_a6f733cd68a255bc3547.png with policy now +2023-07-12 11:37:43,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_6c225fe157884e440759.png with policy now +2023-07-12 11:37:43,407 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,407 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_b06c0ff3bb6bd189e084.png with policy now +2023-07-12 11:37:43,418 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:37:43,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2770_b32e150d333150b6d60d.png with policy now +2023-07-12 11:37:43,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:37:43,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:37:43,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:37:43,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:37:43,826 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0xxgvpi5-media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 11:37:43,837 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gt2ct0cg-media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 11:37:43,839 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mkjpf5ve-media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 11:37:43,845 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9qhcrga2-media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 11:37:46,227 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:46,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:37:46,292 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 11:37:46,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 11:37:46,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 11:37:46,293 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 11:37:46,294 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:37:46,737 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7f5kywss-media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 11:37:46,737 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j9du9k8n-media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 11:37:46,757 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pcf2bkya-media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 11:37:46,768 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hhomi6ll-media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 11:37:49,061 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:37:49,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:37:49,062 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:37:52,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:53,447 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:37:57,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:37:58,309 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:02,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:02,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:02,212 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:02,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:03,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:04,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:04,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:04,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:08,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:10,093 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:38:10,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:10,332 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:12,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:12,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:12,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:12,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:13,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:18,917 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:19,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:19,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:19,590 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:22,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:22,350 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:22,523 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:22,523 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:22,524 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:22,525 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:24,526 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:30,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:33,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:33,672 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:33,673 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:33,675 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:34,369 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:34,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:35,351 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:35,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:35,352 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:35,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:40,095 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:38:40,806 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:43,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:43,461 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:43,461 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:43,463 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:46,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:46,642 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:46,643 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:38:50,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:38:50,642 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:38:50,643 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:38:52,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:54,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:38:54,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:38:54,069 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:38:54,071 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:38:57,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:38:58,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:38:58,661 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:39:02,837 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:04,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:39:04,216 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:39:04,216 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:39:04,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:39:06,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:06,208 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:06,209 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:08,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:10,096 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:39:10,679 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:39:10,680 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:39:14,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:14,723 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:39:14,724 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:39:14,724 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:39:14,726 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:39:19,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:19,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:39:19,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:39:19,623 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:39:19,625 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:39:21,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:21,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:21,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:22,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:39:22,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:39:25,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:30,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:35,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:36,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:36,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:36,307 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:40,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:39:41,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:46,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:51,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:39:51,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:39:51,435 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:39:51,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:39:56,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:01,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:06,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:06,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:06,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:06,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:10,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:40:12,099 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:14,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,328 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_37df37da56d5d142423a.png with policy now +2023-07-12 11:40:14,337 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,337 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_0ebabc283025333f43a1.png with policy now +2023-07-12 11:40:14,347 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,347 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_5cdc5d44821692ab8816.png with policy now +2023-07-12 11:40:14,357 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:40:14,357 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2780_37cb82e9b9e53c9e5c5d.png with policy now +2023-07-12 11:40:14,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:14,397 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:14,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:14,401 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:14,714 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yw4ld93z-media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 11:40:14,720 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ra7y4264-media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 11:40:14,731 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x0gs04uv-media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 11:40:14,792 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yvac02p2-media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 11:40:17,860 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:21,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:21,521 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:21,522 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:22,934 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:23,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:23,028 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:40:23,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 11:40:23,029 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 11:40:23,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 11:40:23,030 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 11:40:23,031 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:40:23,453 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ahdvoobk-media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 11:40:23,456 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o8el157e-media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 11:40:23,475 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tgdtcqou-media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 11:40:23,565 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yqfz6szf-media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 11:40:24,371 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:28,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:33,812 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:34,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:34,391 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:34,392 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:34,394 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:35,047 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:35,048 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:40:36,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:36,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:36,641 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:39,067 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:40,100 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:40:44,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:44,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:44,434 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:44,435 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:44,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:47,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:47,069 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:40:49,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:51,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:40:51,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:40:51,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:40:54,478 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:40:54,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:40:54,480 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:40:54,480 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:40:54,482 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:40:59,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:40:59,091 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:00,376 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:04,476 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:04,478 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:04,479 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:04,481 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:05,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:06,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:06,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:06,640 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:10,101 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:41:11,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:11,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:11,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:14,803 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:14,804 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:14,805 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:14,806 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:16,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:22,102 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:22,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:22,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:22,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:23,381 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:23,382 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:25,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:25,252 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:25,253 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:25,255 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:27,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:32,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:34,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:34,934 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:34,934 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:34,936 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:35,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:35,401 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:37,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:37,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:37,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:37,645 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:40,102 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:41:42,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:45,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:45,569 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:45,569 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:45,571 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:47,418 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:47,419 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:41:48,101 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:50,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:41:50,280 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:41:50,280 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:41:50,282 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:41:52,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:41:52,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:41:52,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:41:53,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:58,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:41:59,437 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:41:59,438 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:42:03,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:07,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:07,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:07,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:08,359 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:10,103 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:42:14,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:19,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:22,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:22,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:22,264 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:24,332 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:29,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:34,333 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:37,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:37,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:37,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:39,399 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:40,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:42:44,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,299 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_5a82d28324cd9b502db5.png with policy now +2023-07-12 11:42:44,308 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,309 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_1df5c10fc9ddab2abd27.png with policy now +2023-07-12 11:42:44,318 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,319 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_a3c09a1cdfb2790cb544.png with policy now +2023-07-12 11:42:44,328 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:42:44,329 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2790_b0ea698af085bca141c6.png with policy now +2023-07-12 11:42:44,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:42:44,371 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:42:44,372 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:42:44,379 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:42:44,630 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0nfjq7b3-media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 11:42:44,632 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/39bbt4ol-media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 11:42:44,660 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nat74ho9-media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 11:42:44,677 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oo900y59-media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 11:42:45,381 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:47,799 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:42:47,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 11:42:47,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 11:42:47,800 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 11:42:47,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 11:42:47,801 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:42:48,169 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/47c50e8j-media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 11:42:48,185 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/o2a6nqej-media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 11:42:48,205 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/grjckrpy-media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 11:42:48,209 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ut37vrgs-media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 11:42:50,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:52,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:42:52,341 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:42:52,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:42:54,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:42:56,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:42:59,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:02,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:03,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:03,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:03,799 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:03,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:07,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:07,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:07,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:08,546 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:10,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:43:11,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:11,836 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:13,466 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:13,467 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:13,467 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:13,469 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:14,470 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:20,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:22,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:22,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:22,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:23,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:23,858 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:24,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:24,308 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:24,308 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:24,310 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:25,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:30,879 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:34,224 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:34,225 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:34,225 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:34,228 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:35,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:35,884 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:36,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:37,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:37,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:37,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:40,106 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:43:41,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:44,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:44,010 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:44,010 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:44,012 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:47,406 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:48,157 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:43:48,158 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:43:52,674 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:43:52,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:43:52,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:43:52,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:43:54,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:43:54,638 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:43:54,639 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:43:54,640 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:43:58,465 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:00,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:44:00,176 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:44:04,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:05,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:44:05,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:44:05,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:44:05,154 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:44:07,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:07,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:07,840 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:10,075 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:10,107 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:44:12,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:44:12,193 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:44:15,528 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:44:15,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:15,529 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:44:15,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:44:15,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:44:19,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:44:19,874 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:44:19,874 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:44:19,876 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:44:21,490 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:22,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:22,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:22,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:24,212 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:44:24,213 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:44:26,982 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:31,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:36,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:37,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:37,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:37,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:40,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:44:42,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:47,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:52,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:44:53,088 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:44:53,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:44:53,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:44:57,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:02,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:07,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:08,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:08,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:08,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:10,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:45:13,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:13,884 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,884 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_0a4eb1c4607d5a5fd63e.png with policy now +2023-07-12 11:45:13,893 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,894 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_a4a930c3aca9ed6b1e66.png with policy now +2023-07-12 11:45:13,906 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_1e810046ab391ec21457.png with policy now +2023-07-12 11:45:13,917 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:45:13,917 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2800_55e6ce392e83cd6c9775.png with policy now +2023-07-12 11:45:13,961 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:13,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:13,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:13,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:14,265 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0jclxal7-media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 11:45:14,272 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/la7oxeap-media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 11:45:14,274 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6copradp-media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 11:45:14,276 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7q8glz2j-media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 11:45:18,733 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:23,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:23,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:23,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:23,807 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:23,808 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:24,555 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:45:24,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:45:24,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 11:45:24,556 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 11:45:24,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 11:45:24,557 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 11:45:24,558 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:45:24,953 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/aeud6s0x-media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 11:45:24,958 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/cou4mf2w-media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 11:45:24,963 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xc50d5pf-media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 11:45:24,969 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/yw17wqiy-media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 11:45:29,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:32,902 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:32,903 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:32,904 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:32,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:34,907 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:36,576 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:45:36,577 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:45:38,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:38,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:38,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:40,110 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:45:40,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:42,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:42,734 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:42,734 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:42,736 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:45,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:48,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:45:48,594 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:45:51,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:45:52,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:45:52,811 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:45:52,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:45:52,813 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:45:53,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:45:53,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:45:53,262 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:45:56,494 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:00,613 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:00,614 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:02,392 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:02,828 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:02,830 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:02,831 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:02,832 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:07,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:08,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:08,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:08,263 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:10,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:46:12,632 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:12,633 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:13,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:13,436 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:13,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:13,437 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:13,439 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:19,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:23,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:23,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:23,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:24,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:24,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:24,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:24,299 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:24,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:24,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:24,894 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:29,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:34,603 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:34,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:34,605 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:34,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:35,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:36,916 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:36,917 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:39,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:39,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:39,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:40,112 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:46:41,327 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:46,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:46,155 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:46,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:46,157 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:47,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:48,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:46:48,933 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:46:50,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:46:50,812 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:46:50,812 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:46:50,814 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:46:52,514 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:46:54,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:46:54,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:46:54,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:46:57,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:00,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:47:00,952 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:47:02,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:07,874 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:09,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:09,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:10,113 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:47:13,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:18,115 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:23,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:24,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:47:24,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:24,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:28,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:33,876 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:38,877 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:39,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:39,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:47:39,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:40,114 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:47:44,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:45,063 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,063 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_79746b73a2d51ab398aa.png with policy now +2023-07-12 11:47:45,072 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,072 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_8fd491043a269feded86.png with policy now +2023-07-12 11:47:45,082 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,082 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_8e4329d82c9013ca58f7.png with policy now +2023-07-12 11:47:45,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:47:45,092 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2810_1f22abcfa3a067c7928c.png with policy now +2023-07-12 11:47:45,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:47:45,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:47:45,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:47:45,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:47:45,512 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nc7vq0w3-media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 11:47:45,539 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ch8yrr8q-media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 11:47:45,546 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f47pc7cq-media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 11:47:45,556 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ywn97eog-media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 11:47:49,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:47:49,266 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 11:47:49,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 11:47:49,267 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 11:47:49,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 11:47:49,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:47:49,722 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eku1kuri-media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 11:47:49,729 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3ta44jei-media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 11:47:49,768 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9t8jlhw7-media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 11:47:49,810 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ltk10z3i-media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 11:47:49,819 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:47:54,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:47:54,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:47:54,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:47:55,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:47:55,010 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:00,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:01,280 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:04,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:04,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:04,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:04,039 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:06,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:09,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:09,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:10,115 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:48:11,116 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:13,300 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:13,301 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:14,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:14,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:14,149 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:14,151 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:16,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:21,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:24,635 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:24,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:24,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:24,638 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:24,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:24,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:25,320 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:25,321 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:27,444 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:32,555 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:34,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:34,962 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:34,962 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:34,964 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:37,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:37,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:38,502 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:39,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:39,771 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:40,116 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:48:43,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:45,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:45,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:45,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:45,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:48:49,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:49,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:48:49,611 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:48:54,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:48:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:48:54,906 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:48:54,907 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:48:55,468 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:48:55,470 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:48:55,471 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:48:55,472 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:00,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:01,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:49:01,627 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:49:05,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:06,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:49:06,308 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:49:06,308 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:49:06,310 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:10,117 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:49:10,731 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:10,732 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:10,732 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:10,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:13,644 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:49:13,645 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:49:15,974 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:49:15,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:15,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:49:15,976 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:49:15,978 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:20,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:49:20,671 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:49:20,671 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:49:20,673 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:49:21,140 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:25,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:49:25,664 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:49:25,755 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:25,756 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:25,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:26,846 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:31,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:36,847 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:40,118 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:49:40,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:40,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:40,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:41,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:46,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:51,931 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:49:55,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:49:55,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:49:55,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:49:56,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:01,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:06,972 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:10,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:50:10,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:10,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:10,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:12,709 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:15,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_0393c6fc4db3e318d3d3.png with policy now +2023-07-12 11:50:15,029 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,030 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_4fee48092f2ede4f3c25.png with policy now +2023-07-12 11:50:15,040 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_ba69b7651ab0dcd73bdc.png with policy now +2023-07-12 11:50:15,050 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:50:15,050 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2820_af052cd0179c44de7cb7.png with policy now +2023-07-12 11:50:15,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:15,091 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:15,091 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:15,096 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:15,341 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ccf3zvw8-media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 11:50:15,343 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/07h7keny-media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 11:50:15,363 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qg7wchih-media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 11:50:15,378 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ifg0kdz2-media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 11:50:17,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:23,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:24,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:25,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:25,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:25,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:25,999 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:50:26,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:50:26,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 11:50:26,000 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 11:50:26,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 11:50:26,001 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 11:50:26,002 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:50:26,343 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0ulagkxk-media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 11:50:26,344 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/m8fxd9dx-media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 11:50:26,346 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/x3xz7y70-media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 11:50:26,375 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z0e008uj-media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 11:50:29,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:34,404 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:34,778 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:34,779 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:34,780 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:34,782 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:50:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:50:40,120 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:50:40,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:40,849 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:40,850 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:40,850 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:45,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:45,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:45,459 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:45,459 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:45,461 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:50,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:50:50,038 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:50:51,232 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:50:56,016 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:50:56,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:50:56,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:50:56,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:50:56,069 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:50:56,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:50:56,072 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:50:57,073 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:02,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:02,057 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:02,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:06,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:06,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:06,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:06,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:08,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:10,121 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:51:11,048 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:11,049 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:11,049 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:51:13,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:14,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:14,075 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:16,932 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:16,933 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:16,933 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:16,935 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:18,936 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:24,147 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:26,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:26,341 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:26,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:26,513 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:26,514 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:51:27,081 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:27,081 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:27,082 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:27,084 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:29,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:34,913 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:37,426 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:37,427 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:37,428 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:37,430 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:38,360 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:38,361 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:40,123 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:51:40,124 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:42,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:42,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:42,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:51:45,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:47,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:47,655 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:47,655 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:47,656 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:50,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:51:50,377 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:51:51,176 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:52,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:51:52,414 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:51:52,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:51:52,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:51:57,065 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:51:57,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:51:57,243 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:51:57,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:02,292 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:02,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:52:02,392 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:52:07,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:10,124 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:52:12,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:12,396 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:12,397 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:12,461 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:17,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:22,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:27,410 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:27,411 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:27,411 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:28,458 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:33,459 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:38,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:40,125 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:52:42,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:42,569 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:42,570 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:43,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:46,721 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,721 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_a5847a1c9ac576d3c242.png with policy now +2023-07-12 11:52:46,730 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,730 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_d0edf809dce9067d913a.png with policy now +2023-07-12 11:52:46,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_1176c8bf7fff463bf9c6.png with policy now +2023-07-12 11:52:46,749 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:52:46,749 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2830_f3f42317c0356c47ff20.png with policy now +2023-07-12 11:52:46,788 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:52:46,789 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:52:46,789 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:52:46,791 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:52:47,041 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xd4oc6qt-media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 11:52:47,059 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1h3w6sog-media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 11:52:47,076 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/i87ajoq2-media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 11:52:47,204 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xc91uqac-media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 11:52:48,791 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:50,699 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:52:50,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 11:52:50,700 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 11:52:50,701 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 11:52:50,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 11:52:50,702 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:52:51,035 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/74nu2nq5-media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 11:52:51,040 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f1im8kr8-media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 11:52:51,041 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/44as5a81-media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 11:52:51,059 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3trgye8s-media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 11:52:53,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:52:56,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:52:57,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:52:57,900 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:52:57,900 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:52:59,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:02,719 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:04,983 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:05,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:05,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:05,820 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:05,822 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:09,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:10,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:53:12,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:53:12,943 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:12,943 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:53:14,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:14,738 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:15,009 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:15,360 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:15,361 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:15,361 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:15,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:20,320 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:25,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:25,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:25,222 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:25,224 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:26,225 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:26,756 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:26,757 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:27,942 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:27,942 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:53:31,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:35,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:35,798 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:35,798 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:35,800 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:36,801 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:38,776 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:38,777 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:40,127 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:53:42,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:43,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:53:43,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:43,615 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:53:46,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:46,567 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:46,567 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:46,569 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:47,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:51,055 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:53:51,056 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:53:53,287 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:57,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:53:57,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:53:57,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:53:57,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:53:59,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:53:59,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:53:59,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:53:59,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:03,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:54:03,074 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:54:04,657 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:07,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:54:07,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:54:07,171 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:54:07,173 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:54:09,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:10,128 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:54:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:14,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:54:14,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:14,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:15,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:54:15,090 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:54:17,980 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:54:17,982 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:54:17,982 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:54:17,984 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:54:19,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:22,592 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:54:22,594 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:54:22,595 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:54:22,597 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:54:25,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:27,109 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:54:27,110 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:54:29,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:29,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:54:29,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:30,297 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:35,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:40,130 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:54:41,130 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:44,251 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:44,251 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:54:46,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:51,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:56,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:54:59,270 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:54:59,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:54:59,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:01,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:06,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:10,131 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:55:12,132 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:14,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:14,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:14,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:16,657 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,657 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_49d3966f62f54fe9ca62.png with policy now +2023-07-12 11:55:16,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,667 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_a708be3afa6ae1a86e60.png with policy now +2023-07-12 11:55:16,678 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,678 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_7e7433279443a3a2bfa6.png with policy now +2023-07-12 11:55:16,696 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:55:16,696 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2840_2d3f82af7844062f4b35.png with policy now +2023-07-12 11:55:16,737 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:16,739 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:16,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:16,741 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:16,980 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u790hx6o-media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 11:55:16,998 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e3qvq5ik-media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 11:55:17,021 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1ymaqvb1-media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 11:55:17,083 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xxqd9my6-media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 11:55:17,742 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:23,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:26,219 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:27,447 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:55:27,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:55:27,448 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 11:55:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 11:55:27,449 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 11:55:27,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 11:55:27,450 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:55:27,791 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/h9q8rsbr-media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 11:55:27,814 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/s2pxx9du-media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 11:55:27,818 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gsggktdi-media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 11:55:27,821 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2tczu5pt-media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 11:55:28,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:29,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:29,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:29,271 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:34,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:35,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:35,385 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:35,385 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:35,386 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:39,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:39,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:55:39,469 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:55:40,132 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:55:44,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:44,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:44,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:55:44,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:46,051 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:46,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:46,053 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:46,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:50,086 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:51,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:55:51,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:55:55,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:55:56,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:55:56,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:55:56,403 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:55:56,405 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:55:59,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:55:59,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:55:59,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:01,095 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:03,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:03,505 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:06,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:06,242 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:06,242 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:06,243 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:06,245 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:10,135 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:56:12,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:14,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:14,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:14,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:15,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:15,529 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:16,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:16,345 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:16,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:16,348 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:17,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:22,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:26,434 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:26,436 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:26,436 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:26,438 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:27,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:27,797 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:28,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:29,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:29,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:29,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:33,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:36,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:36,388 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:36,388 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:36,390 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:38,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:39,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:39,817 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:40,136 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:56:44,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:44,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:44,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:56:44,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:46,771 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:46,772 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:46,772 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:46,774 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:49,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:51,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:56:51,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:56:51,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:56:51,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:56:51,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:56:51,835 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:56:55,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:56:59,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:56:59,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:56:59,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:00,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:03,857 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:57:05,420 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:10,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:57:11,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:14,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:14,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:14,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:16,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:21,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:26,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:29,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:29,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:29,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:31,483 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:36,484 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:40,138 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:57:42,139 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:44,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:44,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:44,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:45,899 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,900 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_dfe649466fc1947dac85.png with policy now +2023-07-12 11:57:45,908 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,909 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_2975002e2263b7b13da6.png with policy now +2023-07-12 11:57:45,919 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,919 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_462a59d1382cf36adaec.png with policy now +2023-07-12 11:57:45,929 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 11:57:45,929 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2850_9c5803c959d6e590aeae.png with policy now +2023-07-12 11:57:45,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:57:45,972 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:57:45,973 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:57:45,974 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:57:46,247 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w1b9zd0x-media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 11:57:46,259 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/7zunlxji-media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 11:57:46,273 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nhoxa207-media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 11:57:46,289 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a5exp6qt-media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 11:57:47,975 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:52,180 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:57:52,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:57:52,181 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 11:57:52,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 11:57:52,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 11:57:52,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 11:57:52,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 11:57:52,588 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4m4pkq5c-media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 11:57:52,602 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/axex8uqg-media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 11:57:52,615 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/953wek8n-media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 11:57:52,655 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4vbv94n1-media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 11:57:53,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:57:56,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:57:59,366 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:57:59,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:57:59,367 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:57:59,423 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:04,200 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:04,718 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:05,915 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:05,918 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:05,919 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:05,921 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:10,139 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:58:10,142 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:58:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:58:15,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:58:15,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:15,575 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:15,575 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:15,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:15,577 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:16,224 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:16,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:20,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:26,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:26,782 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:26,784 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:26,784 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:26,786 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:28,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:28,241 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:30,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:58:30,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:58:30,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:58:32,093 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:37,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:37,415 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:37,417 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:37,418 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:37,420 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:40,140 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:58:40,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:40,261 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:43,063 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:45,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:58:45,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:58:45,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:58:46,534 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:46,536 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:46,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:46,538 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:48,539 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:52,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:58:52,538 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:58:53,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:58:57,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:58:57,105 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:58:57,106 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:58:57,108 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:58:59,109 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:00,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:00,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:00,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:04,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:04,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:59:04,561 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:59:07,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:59:07,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:59:07,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:59:07,551 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:59:09,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:10,141 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:59:15,007 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:15,008 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:15,008 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:15,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:16,579 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:59:16,580 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:59:18,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:59:18,018 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:59:18,018 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:59:18,020 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:59:20,284 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:22,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 11:59:22,428 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 11:59:22,429 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 11:59:22,431 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 11:59:26,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:28,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 11:59:28,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 11:59:30,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:30,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:30,187 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:31,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:36,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:40,142 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 11:59:42,144 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:45,193 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 11:59:45,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 11:59:45,194 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 11:59:47,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:52,239 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 11:59:57,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:00,206 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:00,207 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:00,207 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:02,249 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:07,250 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:10,144 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:00:13,145 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:15,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:15,368 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:15,369 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:16,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,740 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_c18b3c294fb48283f9a6.png with policy now +2023-07-12 12:00:16,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,755 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_e995c389d61465ef369b.png with policy now +2023-07-12 12:00:16,766 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,766 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_a6d367b1ce47fd112b6d.png with policy now +2023-07-12 12:00:16,776 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:00:16,776 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2860_556f808727f18f3fb7b6.png with policy now +2023-07-12 12:00:16,821 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:16,822 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:16,822 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:16,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:16,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:00:16,947 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:00:16,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:00:16,948 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:00:16,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:00:16,949 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:00:17,055 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rjv23evm-media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:00:17,070 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j4wm7qr7-media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:00:17,071 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/u07zm7bo-media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:00:17,096 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sg7wbrnb-media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:00:17,450 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/14s1hk3v-media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:00:17,456 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/joxct5od-media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:00:17,795 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/12wbqli7-media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:00:17,801 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hkzx7la1-media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:00:18,825 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:24,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:26,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:28,966 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:00:30,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:31,014 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:31,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:31,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:35,867 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:36,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:36,554 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:36,554 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:36,556 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:40,145 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:00:40,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:00:40,986 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:00:41,146 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:46,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:00:46,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:00:46,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:00:46,024 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:46,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:46,120 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:46,122 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:47,123 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:52,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:00:53,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:00:53,005 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:00:55,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:00:55,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:00:55,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:00:55,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:00:58,266 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:01,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:01,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:01,016 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:03,873 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:05,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:05,024 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:06,446 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:06,447 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:06,447 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:06,449 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:09,653 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:10,146 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:01:15,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:16,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:16,475 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:16,476 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:17,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:17,044 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:17,045 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:17,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:17,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:17,295 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:20,627 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:25,887 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:27,091 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:27,092 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:27,093 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:27,095 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:29,312 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:29,313 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:31,608 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:32,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:32,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:32,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:37,062 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:38,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:38,126 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:38,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:38,128 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:40,147 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:01:41,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:41,333 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:42,895 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:47,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:01:47,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:01:47,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:01:48,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:48,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:48,579 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:48,580 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:48,582 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:53,300 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:01:53,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:01:53,302 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:01:53,302 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:01:53,305 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:01:53,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:01:53,351 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:01:58,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:02,069 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:02,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:02,070 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:04,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:05,370 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:02:09,169 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:10,148 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:02:15,149 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:17,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:17,070 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:17,071 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:21,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:26,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:31,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:32,235 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:32,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:32,236 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:36,295 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:40,149 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:02:42,150 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:47,151 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:47,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:02:47,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:02:47,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:02:47,689 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,689 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_bcac2792455fc0c610c6.png with policy now +2023-07-12 12:02:47,698 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,698 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_0e8bce2d57e7e047c357.png with policy now +2023-07-12 12:02:47,708 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,708 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_fe5c449ea1876a5e69ca.png with policy now +2023-07-12 12:02:47,718 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:02:47,718 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2870_1d174f0fc051b4fa542a.png with policy now +2023-07-12 12:02:47,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:02:47,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:02:47,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:02:47,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:02:48,073 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ce8epkca-media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:02:48,116 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ugy3jxxi-media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:02:48,152 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8nq1c9x7-media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:02:48,174 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d8d71ko6-media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:02:52,301 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:53,709 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:02:53,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:02:53,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:02:53,710 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:02:53,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:02:53,711 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:02:53,712 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:02:54,048 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4163l9ir-media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:02:54,057 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dxoas7v0-media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:02:54,094 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0sux7973-media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:02:54,331 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ubfm9zai-media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:02:57,331 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:02:57,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:02,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:02,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:02,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:02,562 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:05,729 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:07,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:07,086 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:07,086 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:07,088 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:08,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:10,152 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:03:13,477 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:16,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:16,235 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:16,236 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:16,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:17,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:17,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:17,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:17,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:17,749 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:19,029 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:24,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:26,431 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:26,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:26,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:26,434 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:29,768 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:29,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:30,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:32,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:32,236 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:32,237 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:36,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:37,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:37,440 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:37,440 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:37,442 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:40,151 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:03:41,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:41,790 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:42,339 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:47,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:03:47,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:03:47,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:03:47,409 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:47,903 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:47,904 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:47,905 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:47,906 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:03:52,743 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:54,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:03:54,068 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:03:57,938 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:03:59,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:03:59,390 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:03:59,390 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:03:59,392 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:02,343 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:02,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:02,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:03,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:06,086 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:04:06,087 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:04:08,759 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:09,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:04:09,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:04:09,185 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:04:09,187 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:10,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:04:14,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:17,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:17,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:17,344 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:18,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:04:18,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:04:19,311 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:04:19,313 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:19,313 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:04:19,314 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:04:19,316 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:23,634 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:04:23,636 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:04:23,636 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:04:23,637 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:04:25,245 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:30,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:04:30,124 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:04:30,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:32,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:32,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:32,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:35,589 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:40,153 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:04:41,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:46,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:47,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:04:47,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:04:47,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:04:51,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:04:56,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:01,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:02,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:02,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:02,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:07,540 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:10,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:05:13,158 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:17,498 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:17,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:17,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:18,098 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,099 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_9b8a7d4902a70bbe3529.png with policy now +2023-07-12 12:05:18,108 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,108 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_948197ed3bdc8486cb6a.png with policy now +2023-07-12 12:05:18,119 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,119 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_7ab6e23c2268ebda9ab0.png with policy now +2023-07-12 12:05:18,129 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:05:18,129 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2880_ad77bbc22d61066c6ce9.png with policy now +2023-07-12 12:05:18,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:18,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:18,170 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:18,170 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:18,172 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:05:18,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:05:18,484 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:05:18,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:05:18,485 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:05:18,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:05:18,486 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:05:18,533 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/z857hs5f-media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:05:18,534 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fimltdk8-media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:05:18,540 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k2ckmw89-media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:05:18,551 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/jhlu0xn2-media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:05:18,912 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ezs2obf-media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:05:18,925 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zn1z5r4i-media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:05:18,972 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/htsbku3n-media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:05:19,319 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9b0brssr-media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:05:23,263 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:27,670 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:28,671 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:30,504 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:05:32,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:32,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:32,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:34,197 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:36,462 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:36,463 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:36,463 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:36,464 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:05:39,953 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:40,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:05:42,523 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:05:42,524 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:05:45,833 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:45,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:45,970 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:45,970 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:45,972 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:05:48,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:05:48,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:05:48,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:05:51,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:54,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:05:54,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:05:56,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:05:56,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:05:56,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:05:56,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:05:56,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:02,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:03,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:03,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:03,003 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:06,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:06,562 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:07,373 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:07,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:07,374 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:07,374 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:07,376 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:10,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:06:12,516 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:17,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:17,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:17,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:17,425 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:18,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:18,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:18,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:18,111 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:18,833 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:18,834 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:23,479 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:27,658 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:27,660 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:27,660 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:27,662 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:28,663 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:30,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:30,853 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:33,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:33,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:33,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:34,106 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:37,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:37,781 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:37,782 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:37,784 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:39,785 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:40,157 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:06:42,868 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:42,869 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:44,904 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:47,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:47,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:47,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:47,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:48,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:06:48,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:06:48,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:06:50,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:06:52,298 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:06:52,299 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:06:52,300 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:06:52,302 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:06:54,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:06:54,886 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:06:55,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:00,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:03,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:03,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:03,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:06,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:10,159 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:07:12,160 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:17,161 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:18,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:18,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:18,189 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:22,259 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:27,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:32,260 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:33,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:33,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:33,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:37,294 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:40,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:07:43,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:46,719 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,719 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_cb1bd764507bb8c18d8a.png with policy now +2023-07-12 12:07:46,801 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_8578482fbd741b1fda1e.png with policy now +2023-07-12 12:07:46,814 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,814 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_98e0fea80621818b19b9.png with policy now +2023-07-12 12:07:46,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:07:46,824 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2890_5371e29c2e2d3a7d3242.png with policy now +2023-07-12 12:07:46,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:07:46,864 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:07:46,864 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:07:46,866 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:07:47,159 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/txe81qoa-media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:07:47,229 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j7ihcke2-media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:07:47,265 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0wx8ct10-media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:07:47,272 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/canyduet-media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:07:48,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:07:48,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:07:48,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:07:48,269 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:54,156 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:07:55,234 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:07:55,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:07:55,235 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:07:55,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:07:55,236 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:07:55,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:07:55,237 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:07:55,660 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mzblicci-media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:07:55,670 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/4t8luk82-media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:07:55,694 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/iwa1v40l-media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:07:55,714 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kxp6z904-media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:07:56,805 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:07:59,262 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:03,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:03,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:03,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:04,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:06,875 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:06,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:06,877 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:06,879 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:07,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:07,254 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:10,160 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:08:10,162 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:15,816 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:16,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:16,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:16,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:16,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:18,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:18,264 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:18,265 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:19,275 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:19,276 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:20,842 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:26,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:26,744 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:26,745 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:26,745 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:26,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:31,385 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:31,386 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:31,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:33,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:33,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:33,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:37,056 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:37,058 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:37,058 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:37,058 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:37,061 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:40,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:08:42,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:43,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:43,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:47,133 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:47,134 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:47,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:47,137 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:48,138 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:49,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:08:49,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:08:49,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:08:54,017 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:08:55,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:08:55,769 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:08:57,194 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:08:57,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:08:57,196 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:08:57,199 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:08:59,200 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:04,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:04,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:04,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:04,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:07,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:09:07,647 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:09:07,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:09:07,649 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:09:07,788 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:09:07,789 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:09:09,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:10,163 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:09:15,430 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:18,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:09:18,353 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:09:18,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:09:18,355 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:09:19,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:19,105 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:19,105 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:19,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:09:19,806 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:09:20,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:23,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:09:23,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:09:23,190 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:09:23,192 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:09:26,864 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:31,825 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:09:31,826 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:09:31,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:34,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:34,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:34,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:37,261 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:40,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:09:43,165 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:48,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:49,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:09:49,137 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:09:49,138 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:09:53,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:09:58,256 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:03,257 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:04,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:04,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:04,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:08,291 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:10,165 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:10:14,166 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:16,945 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,945 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_3d0ad7c57f2af82f8bbc.png with policy now +2023-07-12 12:10:16,955 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,955 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_8f1fe1b72bfe04a0b014.png with policy now +2023-07-12 12:10:16,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,965 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_aa4b86f6ab2a980e6525.png with policy now +2023-07-12 12:10:16,975 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:10:16,975 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2900_27c9042e9f887913b3c0.png with policy now +2023-07-12 12:10:17,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:17,015 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:17,015 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:17,017 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:10:17,394 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8chtjdl5-media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:10:17,398 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hjwj7aji-media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:10:17,400 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/fmb4lavj-media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:10:17,418 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ydejr775-media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:10:19,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:19,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:19,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:19,308 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:20,149 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:10:20,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:10:20,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:10:20,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:10:20,151 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:10:20,152 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:10:20,667 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5h6e567u-media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:10:20,669 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/g7z8b3qz-media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:10:20,670 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/q5jqkch9-media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:10:20,675 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5cosu0hu-media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:10:24,349 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:27,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:29,522 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:32,170 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:10:34,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:34,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:34,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:35,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:36,739 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:36,740 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:36,740 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:36,742 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:10:40,167 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:10:41,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:44,191 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:10:44,192 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:10:46,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:46,414 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:46,416 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:46,416 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:46,418 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:10:49,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:10:49,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:10:49,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:10:51,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:56,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:10:56,209 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:10:56,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:10:56,541 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:10:56,541 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:10:56,542 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:10:56,544 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:02,342 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:04,214 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:04,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:04,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:06,754 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:06,755 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:06,756 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:06,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:07,758 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:08,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:08,229 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:10,168 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:11:13,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:16,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:16,342 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:16,342 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:16,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:18,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:19,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:19,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:19,215 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:20,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:20,492 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:23,945 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:26,878 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:26,880 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:26,880 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:26,882 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:29,505 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:32,510 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:32,511 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:34,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:34,433 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:34,434 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:34,991 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:37,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:37,664 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:37,664 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:37,666 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:40,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:11:40,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:44,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:44,530 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:45,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:47,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:47,419 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:47,419 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:47,421 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:49,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:11:49,472 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:11:49,473 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:11:51,028 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:11:52,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:11:52,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:11:52,218 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:11:52,220 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:11:56,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:11:56,553 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:11:56,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:01,845 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:04,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:04,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:04,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:07,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:10,170 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:12:13,171 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:18,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:19,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:19,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:19,574 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:23,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:28,636 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:33,637 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:34,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:34,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:34,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:38,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:40,171 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:12:44,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:46,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,363 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_2d10951a0f8b9aed5734.png with policy now +2023-07-12 12:12:46,372 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,372 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_42b541158c5e9a2b90ee.png with policy now +2023-07-12 12:12:46,382 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,382 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_ba216e11217e75e9719b.png with policy now +2023-07-12 12:12:46,392 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:12:46,393 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2910_885722981f42772a687d.png with policy now +2023-07-12 12:12:46,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:12:46,433 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:12:46,433 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:12:46,436 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:12:46,676 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sijfrl8w-media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:12:46,680 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/riw9ty4b-media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:12:46,704 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/w258tvlk-media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:12:46,725 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8ehpj260-media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:12:49,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:49,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:12:49,600 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:12:49,601 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:12:54,299 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:12:56,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:12:56,903 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:12:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:12:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:12:56,904 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:12:56,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:12:56,905 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:12:56,906 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:12:57,277 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/46b39vmh-media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:12:57,284 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hg5oa619-media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:12:57,300 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mwtdoh5z-media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:12:57,303 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tl0qjqrx-media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:12:59,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:04,978 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:04,979 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:04,979 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:05,031 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:05,544 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:05,546 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:05,546 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:05,548 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:08,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:08,923 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:10,172 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:13:10,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:15,432 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:15,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:15,838 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:15,838 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:15,840 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:20,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:20,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:20,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:20,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:20,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:20,945 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:25,340 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:25,341 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:25,341 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:25,343 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:26,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:31,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:32,964 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:32,965 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:35,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:35,647 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:35,647 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:36,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:36,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:36,211 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:36,213 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:37,213 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:40,174 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:13:42,321 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:44,984 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:46,628 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:46,629 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:46,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:46,632 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:47,633 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:51,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:13:51,377 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:13:51,378 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:13:53,000 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:13:56,990 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:13:56,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:13:56,992 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:13:56,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:13:57,268 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:13:57,269 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:13:58,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:04,899 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:06,456 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:06,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:06,457 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:06,464 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:14:06,493 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:14:06,493 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:14:06,496 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:14:09,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:14:09,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:14:09,986 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:10,175 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:14:15,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:17,293 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:14:17,295 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:14:17,295 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:14:17,296 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:14:21,157 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:21,422 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:14:21,423 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:14:21,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:21,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:21,706 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:22,278 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:14:22,279 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:14:22,279 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:14:22,281 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:14:26,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:31,863 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:33,440 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:14:33,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:14:36,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:36,706 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:36,707 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:37,748 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:40,176 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:14:43,177 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:48,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:51,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:14:51,815 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:14:51,816 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:14:53,870 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:14:58,871 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:03,872 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:06,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:06,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:06,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:09,013 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:10,178 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:15:14,179 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:16,515 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,515 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_3f89f7c3f5ff7126de27.png with policy now +2023-07-12 12:15:16,524 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,524 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_ec96bb36cf4e66588cd4.png with policy now +2023-07-12 12:15:16,533 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,533 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_e0997462fe5eec3ca9d3.png with policy now +2023-07-12 12:15:16,543 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:15:16,543 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2920_0a30f7284bacae8356b3.png with policy now +2023-07-12 12:15:16,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:16,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:16,583 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:16,586 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:16,831 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zmipigev-media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:15:16,844 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/bczgeohh-media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:15:16,869 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/471spo3a-media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:15:16,937 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6ioot3bx-media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:15:19,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:21,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:15:21,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:21,772 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:15:21,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:15:21,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:15:21,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:15:21,774 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:15:21,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:21,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:21,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:22,145 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qbc8kinp-media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:15:22,157 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/k7offw2l-media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:15:22,159 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/blzf4ess-media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:15:22,160 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/uck3bxjx-media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:15:25,390 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:26,995 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:31,148 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:33,791 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:36,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:36,185 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:36,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:36,186 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:36,188 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:36,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:36,968 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:36,969 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:40,179 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:15:41,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:45,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:15:45,810 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:45,963 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:45,965 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:45,965 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:45,968 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:47,969 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:53,042 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:15:53,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:15:53,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:15:53,092 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:15:56,450 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:15:56,452 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:15:56,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:15:56,454 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:15:57,827 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:15:57,828 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:15:58,455 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:04,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:07,421 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:07,423 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:07,423 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:07,425 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:08,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:08,043 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:08,043 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:09,847 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:09,848 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:09,971 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:10,180 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:16:15,244 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:17,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:17,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:17,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:17,844 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:20,367 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:22,132 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:22,133 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:23,125 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:23,126 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:23,126 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:25,725 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:28,632 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:28,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:28,633 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:28,635 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:31,348 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:34,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:34,150 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:36,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:38,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:38,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:38,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:38,948 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:38,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:38,950 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:38,952 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:40,181 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:16:41,798 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:46,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:46,169 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:47,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:48,599 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:48,600 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:48,600 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:48,601 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:53,066 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:16:53,248 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:16:53,250 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:16:53,250 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:16:53,251 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:16:53,382 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:16:53,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:16:53,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:16:58,188 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:16:58,189 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:16:58,840 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:03,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:08,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:17:08,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:08,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:09,040 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:10,182 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:17:14,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:19,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:23,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:23,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:17:23,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:24,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:29,438 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:34,439 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:38,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:38,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:17:38,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:40,183 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:17:40,186 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:45,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:47,747 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,747 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_fd71982266285f250c3c.png with policy now +2023-07-12 12:17:47,757 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,757 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_2268465f1770440e1c31.png with policy now +2023-07-12 12:17:47,768 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,768 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_94f80d2b1ffdc85aabfa.png with policy now +2023-07-12 12:17:47,778 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:17:47,778 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2930_ccf35b7fa382262ad056.png with policy now +2023-07-12 12:17:47,818 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:17:47,819 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:17:47,819 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:17:47,821 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:17:48,086 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dljh5a3o-media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:17:48,092 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/mv5p4tkr-media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:17:48,112 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/a2m9e8ia-media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:17:48,118 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ho5wvbyg-media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:17:50,822 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:53,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:17:53,383 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:17:56,104 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:17:57,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:17:58,541 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:17:58,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:17:58,542 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:17:58,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:17:58,543 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:17:58,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:17:58,544 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:17:58,892 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wqt4ae0r-media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:17:58,902 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/en49xxp1-media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:17:58,910 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xp40gmfc-media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:17:58,917 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eu5dnif0-media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:18:01,909 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:06,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:06,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:06,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:06,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:07,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:08,383 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:08,384 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:10,184 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:18:10,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:10,563 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:12,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:16,989 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:16,991 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:16,991 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:16,993 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:17,994 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:22,587 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:22,588 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:23,765 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:24,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:18:24,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:24,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:27,413 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:27,415 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:27,415 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:27,417 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:29,418 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:34,606 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:34,607 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:34,881 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:39,074 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:39,075 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:39,075 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:39,077 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:39,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:18:39,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:39,153 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:40,185 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:18:40,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:45,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:46,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:46,626 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:18:49,457 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:18:49,458 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:18:49,458 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:18:49,460 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:18:50,460 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:54,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:18:54,152 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:18:54,152 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:18:55,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:18:58,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:18:58,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:00,163 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:00,164 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:00,164 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:00,166 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:01,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:06,729 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:09,271 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:09,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:09,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:09,839 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:09,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:09,841 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:09,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:10,186 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:19:10,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:19:10,912 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:12,187 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:17,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:20,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:20,052 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:20,052 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:20,055 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:22,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:22,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:19:22,932 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:24,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:24,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:24,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:24,757 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:19:24,758 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:19:24,758 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:19:24,760 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:19:28,485 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:33,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:34,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:19:34,950 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:19:38,486 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:39,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:39,272 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:39,272 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:40,187 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:19:44,188 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:49,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:54,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:19:54,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:19:54,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:19:54,322 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:19:59,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:04,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:09,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:09,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:09,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:09,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:10,188 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:20:15,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:18,857 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,857 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_11bcda97ccccef6067ba.png with policy now +2023-07-12 12:20:18,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,867 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_ba511e69b1b4757bb114.png with policy now +2023-07-12 12:20:18,877 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,877 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_32467ef9d42b5e1e1a33.png with policy now +2023-07-12 12:20:18,888 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:20:18,888 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2940_c49f0237624dac615f66.png with policy now +2023-07-12 12:20:18,929 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:18,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:18,930 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:18,932 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:19,182 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/eaiwd0yw-media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:20:20,218 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kmxnke0i-media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:20:20,521 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/az3mw0ve-media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:20:20,619 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/e7bsjeyd-media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:20:20,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:23,287 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:20:23,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:20:23,288 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:20:23,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:20:23,289 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:20:23,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:20:23,290 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:20:23,664 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/xwfdjk5j-media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:20:23,665 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/5oqg297a-media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:20:23,666 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/9bkxkooz-media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:20:23,667 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6iy29cl2-media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:20:24,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:24,323 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:24,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:26,374 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:28,307 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:31,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:35,306 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:20:36,977 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:37,361 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:37,363 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:37,363 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:37,365 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:39,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:39,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:39,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:40,189 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:20:42,937 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:47,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:47,078 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:47,078 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:47,081 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:47,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:20:47,330 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:20:48,082 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:53,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:55,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:20:55,077 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:20:55,077 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:20:58,055 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:20:58,057 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:20:58,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:20:58,059 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:20:59,060 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:20:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:20:59,349 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:04,946 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:08,988 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:08,990 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:08,990 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:08,992 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:09,993 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:10,190 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:21:11,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:11,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:11,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:11,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:11,367 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:15,412 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:19,036 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:19,038 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:19,038 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:19,040 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:21,041 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:23,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:23,636 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:26,131 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:26,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:26,267 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:26,268 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:29,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:29,161 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:29,161 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:29,163 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:31,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:35,658 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:35,659 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:36,696 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:39,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:39,538 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:39,538 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:39,540 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:40,191 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:21:41,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:41,493 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:41,494 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:42,050 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:47,189 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:47,681 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:21:49,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:49,610 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:49,611 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:49,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:53,071 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:54,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:21:54,223 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:21:54,223 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:21:54,225 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:21:56,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:21:56,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:21:56,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:21:58,551 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:21:59,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:21:59,704 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:22:03,552 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:08,553 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:10,192 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:22:11,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:11,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:11,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:13,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:18,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:23,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:26,503 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:26,504 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:26,504 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:29,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:34,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:39,566 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:40,193 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:22:41,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:41,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:41,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:44,701 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:48,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,209 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_5189a1941e0ee2300ecf.png with policy now +2023-07-12 12:22:48,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,218 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_dce90c9fdac2309feb19.png with policy now +2023-07-12 12:22:48,228 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,228 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_0d199df0c17d88c8726a.png with policy now +2023-07-12 12:22:48,237 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:22:48,237 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2950_626d6d1dadf7aad42361.png with policy now +2023-07-12 12:22:48,276 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:22:48,278 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:22:48,278 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:22:48,280 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:22:48,566 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rr17uxev-media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:22:48,582 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hckqtfvy-media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:22:48,589 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gbakifvj-media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:22:48,622 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/3myhupc6-media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:22:50,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:55,290 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:22:56,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:22:56,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:22:56,616 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:22:57,618 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:00,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:00,072 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:23:00,073 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:23:00,428 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kntt3qh8-media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:23:00,438 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/css5rft9-media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:23:00,441 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/82lwht82-media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:23:00,463 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/rualo752-media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:23:01,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:06,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:06,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:06,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:06,691 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:06,693 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:10,195 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:23:11,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:23:11,631 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:11,632 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:11,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:12,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:12,094 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:16,667 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:16,669 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:16,670 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:16,672 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:17,673 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:23,275 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:24,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:24,116 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:26,630 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:26,630 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:26,841 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:26,841 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:26,842 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:26,843 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:28,844 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:34,280 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:36,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:36,134 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:36,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:36,967 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:36,968 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:36,970 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:39,506 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:40,196 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:23:41,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:23:41,735 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:41,736 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:45,255 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:47,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:47,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:47,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:47,669 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:23:48,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:23:48,154 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:23:51,234 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:56,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:23:57,570 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:23:57,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:23:57,571 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:23:58,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:23:58,137 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:23:58,137 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:23:58,139 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:00,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:24:00,421 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:02,238 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:08,178 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:08,395 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:24:08,396 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:24:08,396 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:24:08,398 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:10,197 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:24:12,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:24:12,441 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:13,389 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:13,451 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:24:13,452 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:13,452 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:24:18,665 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:19,190 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:24:19,192 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:24:19,192 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:24:19,194 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:24,117 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:24,338 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:24:24,340 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:24:24,340 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:24:24,342 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:24:24,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:24:24,460 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:24:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:28,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:24:29,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:34,615 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:36,489 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:24:39,616 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:40,198 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:24:43,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:43,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:24:43,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:24:45,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:50,606 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:55,607 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:24:58,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:24:58,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:01,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:06,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:10,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:25:12,201 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:13,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:13,565 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:17,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:18,613 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,613 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_7b43af642a476832b780.png with policy now +2023-07-12 12:25:18,622 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,622 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_76db5a33fad883eea4bc.png with policy now +2023-07-12 12:25:18,633 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,633 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_366cee911e557f14b4e5.png with policy now +2023-07-12 12:25:18,643 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:25:18,643 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2960_95e48c47882f1ab9ab69.png with policy now +2023-07-12 12:25:18,682 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:18,683 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:18,683 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:18,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:18,920 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/97p627zh-media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:25:18,933 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tu10k09b-media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:25:18,935 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sxrnbcsc-media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:25:18,977 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/omnjhs6f-media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:25:22,795 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:24,820 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:25:24,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:25:24,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:25:24,821 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:25:24,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:25:24,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:25:24,822 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:25:25,188 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1c7qx1xo-media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:25:25,200 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/d036ppk6-media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:25:25,233 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/flsqtx8a-media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:25:25,235 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/dyx1mfgr-media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:25:28,012 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:28,335 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:28,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:25:28,566 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:33,672 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:36,839 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:25:37,154 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:37,156 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:37,156 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:37,158 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:39,159 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:40,200 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:25:43,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:25:43,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:43,751 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:25:44,334 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:47,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:47,016 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:47,017 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:47,019 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:25:48,861 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:25:50,020 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:55,655 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:25:58,427 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:25:58,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:25:58,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:25:58,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:25:58,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:25:58,751 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:25:58,752 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:00,820 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:00,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:00,882 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:06,184 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:08,898 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:08,900 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:08,901 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:08,902 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:10,202 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:26:11,203 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:12,901 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:12,902 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:13,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:26:13,884 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:13,885 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:16,445 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:19,387 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:19,389 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:19,389 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:19,391 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:21,933 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:25,182 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:25,183 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:27,076 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:29,121 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:26:29,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:29,122 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:29,688 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:29,690 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:29,690 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:29,691 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:32,306 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:37,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:37,204 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:37,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:40,203 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:26:40,529 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:40,530 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:40,530 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:40,532 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:42,533 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:44,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:44,127 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:26:47,646 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:49,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:26:49,225 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:26:50,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:50,651 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:50,651 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:50,653 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:52,654 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:55,370 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:26:55,370 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:26:55,370 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:26:55,373 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:26:57,998 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:26:59,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:26:59,218 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:26:59,219 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:01,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:27:01,245 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:27:03,285 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:08,286 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:10,204 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:27:14,205 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:14,228 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:14,229 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:14,229 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:19,281 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:24,282 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:29,283 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:29,385 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:29,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:29,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:34,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:39,429 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:40,205 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:27:44,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:44,386 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:44,386 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:27:44,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:49,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:49,787 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,787 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_c7e2d3867abfa6c74ec7.png with policy now +2023-07-12 12:27:49,797 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,797 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_da274c6af8f1eb4795bd.png with policy now +2023-07-12 12:27:49,807 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,808 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_3fff3d3522577450efef.png with policy now +2023-07-12 12:27:49,818 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:27:49,818 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2970_949364012c78ad26092f.png with policy now +2023-07-12 12:27:49,865 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:27:49,867 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:27:49,867 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:27:49,869 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:27:50,135 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/1jcwp3dw-media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:27:50,137 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/tvxmkdsd-media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:27:50,151 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/oa0qj1wz-media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:27:50,171 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8hkk9nx5-media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:27:54,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:27:59,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:27:59,394 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:27:59,395 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:00,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:00,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:01,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:01,601 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:01,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:28:01,602 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:28:01,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:28:01,603 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:28:01,604 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:28:01,991 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ahvgb5ys-media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:28:01,993 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/kh91i3vt-media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:28:02,002 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zpaz0iny-media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:28:02,007 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/nwup7i5f-media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:28:05,424 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:08,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:08,722 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:08,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:08,724 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:10,206 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:28:11,164 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:13,622 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:13,623 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:15,003 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:28:15,004 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:28:15,004 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:16,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:18,018 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:18,020 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:18,020 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:18,022 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:21,585 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:25,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:25,647 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:27,565 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:28,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:28,576 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:28,576 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:28,578 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:30,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:28:30,473 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:28:30,474 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:33,033 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:37,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:37,667 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:38,604 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:38,605 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:38,605 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:38,606 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:38,608 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:40,207 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:28:44,044 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:45,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:28:45,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:28:45,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:28:48,956 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:48,957 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:48,958 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:48,960 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:28:49,692 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:28:49,693 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:28:49,960 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:55,597 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:28:59,950 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:28:59,951 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:28:59,951 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:28:59,953 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:00,954 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:00,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:00,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:00,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:01,954 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:01,955 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:06,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:10,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:29:11,167 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:29:11,169 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:29:11,169 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:29:11,171 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:12,172 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:13,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:13,975 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:15,965 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:15,966 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:15,966 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:17,274 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:21,547 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:29:21,549 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:29:21,549 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:29:21,575 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:22,576 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:25,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:25,998 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:26,053 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:29:26,055 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:29:26,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:29:26,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:29:27,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:31,089 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:31,090 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:31,090 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:33,182 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:29:38,019 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:29:38,183 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:40,208 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:29:43,209 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:46,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:29:46,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:29:46,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:29:48,252 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:53,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:29:58,253 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:01,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:01,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:01,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:04,246 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:09,247 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:10,209 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:30:15,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:16,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:16,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:16,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:20,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:20,607 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,607 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_86a486a36860b5c6245c.png with policy now +2023-07-12 12:30:20,616 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,616 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_18ea565e486f4bef2bbd.png with policy now +2023-07-12 12:30:20,626 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,626 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_dfadefec7519a6555efa.png with policy now +2023-07-12 12:30:20,635 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:30:20,636 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2980_26cc58965abb0300912c.png with policy now +2023-07-12 12:30:20,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:20,681 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:30:20,682 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:30:20,685 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:30:20,928 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2eb7nl4q-media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:30:20,934 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/j4sqpmdd-media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:30:20,939 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/l8h8fxsf-media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:30:20,941 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ux6ylmrx-media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:30:25,393 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:26,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:30:26,357 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:30:26,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:30:26,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:30:26,358 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:30:26,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:30:26,359 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:30:26,693 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qdrsthso-media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:30:26,721 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/642cduad-media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:30:26,728 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/br5y3njn-media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:30:26,768 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/wn0aw9xq-media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:30:30,704 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:30,705 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:31,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:31,173 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:31,174 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:36,571 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:38,376 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:30:39,797 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:39,799 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:30:39,799 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:30:39,801 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:30:40,210 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:30:42,211 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:46,344 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:30:46,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:30:46,345 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:30:47,905 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:49,582 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:30:49,583 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:30:49,584 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:30:49,585 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:30:50,395 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:30:50,396 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:30:53,136 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:30:58,564 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:00,428 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:00,430 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:00,430 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:00,432 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:02,096 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:02,097 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:02,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:02,415 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:03,662 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:09,168 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:10,211 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:31:10,823 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:10,824 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:10,825 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:10,827 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:14,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:14,436 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:15,127 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:17,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:17,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:17,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:20,861 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:20,862 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:20,863 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:20,863 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:20,865 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:25,962 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:26,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:26,714 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:31,052 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:31,054 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:31,054 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:31,055 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:31,057 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:32,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:32,480 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:32,481 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:36,129 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:38,732 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:38,733 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:40,212 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:31:41,107 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:41,109 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:41,109 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:41,111 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:42,112 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:47,985 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:48,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:31:48,345 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:31:48,346 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:31:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:31:50,753 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:31:51,664 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:51,666 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:51,667 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:51,668 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:53,669 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:31:56,545 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:31:56,547 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:31:56,547 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:31:56,549 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:31:59,143 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:02,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:32:02,773 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:32:03,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:03,536 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:03,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:04,639 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:09,640 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:10,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:32:15,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:18,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:18,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:18,536 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:20,572 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:25,573 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:30,574 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:33,535 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:33,577 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:33,577 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:35,614 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:40,214 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:32:41,215 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:46,216 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:48,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:32:48,609 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:32:48,609 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:32:51,037 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,037 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_d1566b7d6fc6fface55f.png with policy now +2023-07-12 12:32:51,046 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,046 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_3375e10d6215e10e1662.png with policy now +2023-07-12 12:32:51,056 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,056 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_60ce9836f585a7205184.png with policy now +2023-07-12 12:32:51,066 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:32:51,066 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_2990_7a4d6b6bbbebe28cdf93.png with policy now +2023-07-12 12:32:51,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:32:51,105 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:32:51,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:32:51,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:32:51,106 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:32:51,110 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:32:51,111 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:32:51,111 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:32:51,113 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:32:51,392 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/0xlupm4k-media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:32:51,394 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ja8apt89-media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:32:51,395 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/pba6jtoq-media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:32:51,435 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/sh4g5hvy-media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:32:51,807 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/2h42v809-media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:32:51,845 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/ar23nxuk-media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:32:51,904 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zwe5rveu-media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:32:52,113 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:32:52,181 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/v03si8sk-media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:32:57,836 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:00,908 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:02,910 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:03,120 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:03,121 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:03,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:03,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:03,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:08,100 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:10,215 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:33:10,675 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:10,677 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:10,678 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:10,680 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:13,103 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:16,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:16,139 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:18,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:18,567 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:18,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:18,611 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:20,401 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:20,403 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:20,404 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:20,406 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:23,810 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:28,957 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:29,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:29,159 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:30,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:30,931 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:30,931 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:30,933 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:33,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:33,568 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:33,568 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:34,650 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:39,859 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:40,217 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:33:41,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:41,241 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:41,242 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:41,244 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:42,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:42,177 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:44,949 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:48,927 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:33:48,928 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:33:48,928 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:33:50,489 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:33:52,175 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:33:52,177 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:33:52,177 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:33:52,179 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:33:55,467 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:33:55,468 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:33:56,015 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:01,930 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:02,984 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:02,986 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:02,986 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:02,989 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:05,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:05,134 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:05,135 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:07,693 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:08,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:34:08,487 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:34:10,218 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:34:12,914 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:13,463 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:13,464 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:13,464 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:13,466 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:18,006 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:20,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:20,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:20,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:21,506 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:34:21,507 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:34:23,084 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:23,437 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:23,439 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:23,439 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:23,441 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:28,122 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:28,212 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:34:28,213 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:34:28,213 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:34:28,215 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:34:33,997 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:34,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:34:34,526 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:34:35,509 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:35,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:35,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:39,621 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:40,219 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:34:45,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:50,221 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:34:50,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:34:50,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:34:50,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:34:55,578 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:00,579 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:35:05,510 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:35:05,510 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:35:06,561 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:10,220 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:35:12,222 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:17,223 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:20,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:35:20,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:35:20,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:35:22,770 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:22,930 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,931 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_88582cfed2f573568e58.png with policy now +2023-07-12 12:35:22,940 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,940 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png with policy now +2023-07-12 12:35:22,950 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,950 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_400a50fac321ed4c2979.png with policy now +2023-07-12 12:35:22,961 DEBUG SenderThread:3148485 [sender.py:send():369] send: files +2023-07-12 12:35:22,961 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file media/images/validation_3000_72521f709e19fb5db039.png with policy now +2023-07-12 12:35:23,005 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: partial_history +2023-07-12 12:35:23,006 DEBUG SenderThread:3148485 [sender.py:send():369] send: history +2023-07-12 12:35:23,007 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: summary_record +2023-07-12 12:35:23,010 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:35:23,258 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/8zbk6k31-media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:23,259 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/gsj7r3vp-media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:23,269 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/6we7hu00-media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:23,282 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/zudjuajy-media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:26,890 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:35:26,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:35:26,891 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:26,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:26,892 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:26,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_created():272] file/dir created: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:26,893 INFO Thread-12 :3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images +2023-07-12 12:35:27,235 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/qbv86jh1-media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:27,239 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/f39ne2cy-media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:27,255 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/hanrdfub-media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:27,258 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /tmp/tmpmh47ogltwandb/vfkufde4-media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:28,329 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:33,330 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:35,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: keepalive +2023-07-12 12:35:35,721 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: stop_status +2023-07-12 12:35:35,722 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: stop_status +2023-07-12 12:35:39,780 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:40,222 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:35:45,192 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:45,351 DEBUG SenderThread:3148485 [sender.py:send():369] send: exit +2023-07-12 12:35:45,351 INFO SenderThread:3148485 [sender.py:send_exit():574] handling exit code: 1 +2023-07-12 12:35:45,352 INFO SenderThread:3148485 [sender.py:send_exit():576] handling runtime: 45156 +2023-07-12 12:35:45,354 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:35:45,355 INFO SenderThread:3148485 [sender.py:send_exit():582] send defer +2023-07-12 12:35:45,355 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,355 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 0 +2023-07-12 12:35:45,356 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,356 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 0 +2023-07-12 12:35:45,356 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 1 +2023-07-12 12:35:45,356 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,356 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 1 +2023-07-12 12:35:45,357 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,357 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 1 +2023-07-12 12:35:45,357 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 2 +2023-07-12 12:35:45,357 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,357 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 2 +2023-07-12 12:35:45,357 INFO HandlerThread:3148485 [system_monitor.py:finish():190] Stopping system monitor +2023-07-12 12:35:45,358 DEBUG SystemMonitor:3148485 [system_monitor.py:_start():166] Finished system metrics aggregation loop +2023-07-12 12:35:45,358 DEBUG SystemMonitor:3148485 [system_monitor.py:_start():170] Publishing last batch of metrics +2023-07-12 12:35:45,358 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined cpu monitor +2023-07-12 12:35:45,359 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined disk monitor +2023-07-12 12:35:45,497 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined gpu monitor +2023-07-12 12:35:45,498 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined memory monitor +2023-07-12 12:35:45,498 INFO HandlerThread:3148485 [interfaces.py:finish():202] Joined network monitor +2023-07-12 12:35:45,499 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,499 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 2 +2023-07-12 12:35:45,499 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 3 +2023-07-12 12:35:45,499 DEBUG SenderThread:3148485 [sender.py:send():369] send: stats +2023-07-12 12:35:45,499 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,500 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 3 +2023-07-12 12:35:45,500 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,500 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 3 +2023-07-12 12:35:45,500 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 4 +2023-07-12 12:35:45,500 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,500 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 4 +2023-07-12 12:35:45,501 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,501 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 4 +2023-07-12 12:35:45,501 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 5 +2023-07-12 12:35:45,501 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,501 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 5 +2023-07-12 12:35:45,501 DEBUG SenderThread:3148485 [sender.py:send():369] send: summary +2023-07-12 12:35:45,506 INFO SenderThread:3148485 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-07-12 12:35:45,506 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,506 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 5 +2023-07-12 12:35:45,507 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 6 +2023-07-12 12:35:45,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,507 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 6 +2023-07-12 12:35:45,507 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,507 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 6 +2023-07-12 12:35:45,507 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 7 +2023-07-12 12:35:45,507 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: status_report +2023-07-12 12:35:45,508 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:45,508 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 7 +2023-07-12 12:35:45,508 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:45,508 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 7 +2023-07-12 12:35:46,237 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 8 +2023-07-12 12:35:46,237 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:46,237 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 8 +2023-07-12 12:35:46,238 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:46,238 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 8 +2023-07-12 12:35:46,238 INFO SenderThread:3148485 [job_builder.py:build():232] Attempting to build job artifact +2023-07-12 12:35:46,240 INFO SenderThread:3148485 [job_builder.py:build():266] no source found +2023-07-12 12:35:46,240 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 9 +2023-07-12 12:35:46,240 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:46,240 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 9 +2023-07-12 12:35:46,241 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:46,241 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 9 +2023-07-12 12:35:46,241 INFO SenderThread:3148485 [dir_watcher.py:finish():359] shutting down directory watcher +2023-07-12 12:35:46,352 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:46,925 INFO SenderThread:3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:35:46,926 INFO SenderThread:3148485 [dir_watcher.py:_on_file_modified():289] file/dir modified: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:35:46,926 INFO SenderThread:3148485 [dir_watcher.py:finish():389] scan: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files +2023-07-12 12:35:46,927 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/config.yaml config.yaml +2023-07-12 12:35:46,927 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json wandb-summary.json +2023-07-12 12:35:46,928 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/requirements.txt requirements.txt +2023-07-12 12:35:46,934 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-metadata.json wandb-metadata.json +2023-07-12 12:35:46,934 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log output.log +2023-07-12 12:35:46,941 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_0838dad447e339fd0fea.png media/images/validation_10_0838dad447e339fd0fea.png +2023-07-12 12:35:46,941 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_a8dfbe0ff96ea5050767.png media/images/validation_10_a8dfbe0ff96ea5050767.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_56c2e575b2b9d92db6f0.png media/images/validation_10_56c2e575b2b9d92db6f0.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_10_3f8ba95016c3d0286ac3.png media/images/validation_10_3f8ba95016c3d0286ac3.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_b1104143852d0cdcfdbc.png media/images/validation_20_b1104143852d0cdcfdbc.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_a492b392b73e859b8a77.png media/images/validation_20_a492b392b73e859b8a77.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_3ac29936b57bab3b60df.png media/images/validation_20_3ac29936b57bab3b60df.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_20_0394766c00f144b38aaf.png media/images/validation_20_0394766c00f144b38aaf.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ae28d8db362e0aa9f573.png media/images/validation_30_ae28d8db362e0aa9f573.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_616207d75a28a906c55b.png media/images/validation_30_616207d75a28a906c55b.png +2023-07-12 12:35:46,942 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_2c0d6dfc2d246283392e.png media/images/validation_30_2c0d6dfc2d246283392e.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_30_ed7f0d8809d791b958b4.png media/images/validation_30_ed7f0d8809d791b958b4.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_beffc069b9ded929df09.png media/images/validation_40_beffc069b9ded929df09.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_83474bbbae95371e4fe1.png media/images/validation_40_83474bbbae95371e4fe1.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_8ede4c40d45be37a3114.png media/images/validation_40_8ede4c40d45be37a3114.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_40_cb5f34fc925cd4802446.png media/images/validation_40_cb5f34fc925cd4802446.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_6c49012791de94816289.png media/images/validation_50_6c49012791de94816289.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_5fc1a953a335a81d0538.png media/images/validation_50_5fc1a953a335a81d0538.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_f43b6d21c133e2a6b003.png media/images/validation_50_f43b6d21c133e2a6b003.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_50_8bd9326976928e17b3ca.png media/images/validation_50_8bd9326976928e17b3ca.png +2023-07-12 12:35:46,943 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_6ae01c6a6a2bfa713e44.png media/images/validation_60_6ae01c6a6a2bfa713e44.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_cfabe222535005b7c73d.png media/images/validation_60_cfabe222535005b7c73d.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ddf1c39281d80912060e.png media/images/validation_60_ddf1c39281d80912060e.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_60_ed363dbde584f9cd8a96.png media/images/validation_60_ed363dbde584f9cd8a96.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_09c22363614ab958c3bc.png media/images/validation_70_09c22363614ab958c3bc.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_eba1f69e2b11ef8f44d3.png media/images/validation_70_eba1f69e2b11ef8f44d3.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_d01dfe2d2aeaa1bec479.png media/images/validation_70_d01dfe2d2aeaa1bec479.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_70_24254896e341a078f630.png media/images/validation_70_24254896e341a078f630.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_64b2c49cff2d363d20ad.png media/images/validation_80_64b2c49cff2d363d20ad.png +2023-07-12 12:35:46,944 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_f3e5374d59497f9be535.png media/images/validation_80_f3e5374d59497f9be535.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_653670180f2536edf28e.png media/images/validation_80_653670180f2536edf28e.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_80_ca894d05d078201593b2.png media/images/validation_80_ca894d05d078201593b2.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_d4b9dc3d41c12fcb9fe7.png media/images/validation_90_d4b9dc3d41c12fcb9fe7.png +2023-07-12 12:35:46,945 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_21a56bd8f7d955ce07a3.png media/images/validation_90_21a56bd8f7d955ce07a3.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_8b813df4910f6e14fe9f.png media/images/validation_90_8b813df4910f6e14fe9f.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_90_b276db68b1c2650b37f5.png media/images/validation_90_b276db68b1c2650b37f5.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_661b1a72c771caa0e001.png media/images/validation_100_661b1a72c771caa0e001.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_85a79c24055b252b391d.png media/images/validation_100_85a79c24055b252b391d.png +2023-07-12 12:35:46,946 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_e3b8328c3efd11dc0868.png media/images/validation_100_e3b8328c3efd11dc0868.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_100_5f675bc00f07f97610c2.png media/images/validation_100_5f675bc00f07f97610c2.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_ed89fce5f94211880d08.png media/images/validation_110_ed89fce5f94211880d08.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_519be1aabb2ce5adb552.png media/images/validation_110_519be1aabb2ce5adb552.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_0a38067c3b4b0763814d.png media/images/validation_110_0a38067c3b4b0763814d.png +2023-07-12 12:35:46,947 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_110_57b921c4856dfbee5311.png media/images/validation_110_57b921c4856dfbee5311.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_b4dbdc5959d71b1b0010.png media/images/validation_120_b4dbdc5959d71b1b0010.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_3aa26b32048f4df01e9f.png media/images/validation_120_3aa26b32048f4df01e9f.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_18e3cfcb4e5db9c7981b.png media/images/validation_120_18e3cfcb4e5db9c7981b.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_120_f3f35ebd04d7e55194ae.png media/images/validation_120_f3f35ebd04d7e55194ae.png +2023-07-12 12:35:46,948 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_3e5c971e8f903ffb6755.png media/images/validation_130_3e5c971e8f903ffb6755.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_b27cf42472a6e5022d7a.png media/images/validation_130_b27cf42472a6e5022d7a.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_db9b8ff059ac75e60033.png media/images/validation_130_db9b8ff059ac75e60033.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_130_1846f2582dc6add0423d.png media/images/validation_130_1846f2582dc6add0423d.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_20276ffa7780c15c8826.png media/images/validation_140_20276ffa7780c15c8826.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_5f10192137f91e3d88e7.png media/images/validation_140_5f10192137f91e3d88e7.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_f11466fe48b33d6f9da9.png media/images/validation_140_f11466fe48b33d6f9da9.png +2023-07-12 12:35:46,949 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_140_8cf36f31bd00d9abfd34.png media/images/validation_140_8cf36f31bd00d9abfd34.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_ad78c61f0f7db6d206fd.png media/images/validation_150_ad78c61f0f7db6d206fd.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_da52d8afd9d0f2869f66.png media/images/validation_150_da52d8afd9d0f2869f66.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_c4f6c02b99e6533a2eb2.png media/images/validation_150_c4f6c02b99e6533a2eb2.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_150_843770c3f4c58f7b4f6c.png media/images/validation_150_843770c3f4c58f7b4f6c.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_897ca9ce40094218e736.png media/images/validation_160_897ca9ce40094218e736.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_a6aefafa2d783d4803ef.png media/images/validation_160_a6aefafa2d783d4803ef.png +2023-07-12 12:35:46,950 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_c063d5e9ec2283745c8b.png media/images/validation_160_c063d5e9ec2283745c8b.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_160_542fde61a9a7764e7963.png media/images/validation_160_542fde61a9a7764e7963.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_7e7561227c49152fdf02.png media/images/validation_170_7e7561227c49152fdf02.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_249e7863a11a9d2249ff.png media/images/validation_170_249e7863a11a9d2249ff.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_582ff9cdb53e3daa2736.png media/images/validation_170_582ff9cdb53e3daa2736.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_170_80a5f2920f4985e1b760.png media/images/validation_170_80a5f2920f4985e1b760.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_bcd830ba6ea2f7b921a8.png media/images/validation_180_bcd830ba6ea2f7b921a8.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_9f041d8a799e7d60ca1c.png media/images/validation_180_9f041d8a799e7d60ca1c.png +2023-07-12 12:35:46,951 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_2942fa312a4899f548b2.png media/images/validation_180_2942fa312a4899f548b2.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_180_57486bf611e806418a3a.png media/images/validation_180_57486bf611e806418a3a.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_1689c770ac115a929d63.png media/images/validation_190_1689c770ac115a929d63.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_18d5cba2a1158df428aa.png media/images/validation_190_18d5cba2a1158df428aa.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_c50d65d1788fc664335f.png media/images/validation_190_c50d65d1788fc664335f.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_190_2b5b9594c7937018b6be.png media/images/validation_190_2b5b9594c7937018b6be.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_50004396bfec8e96747d.png media/images/validation_200_50004396bfec8e96747d.png +2023-07-12 12:35:46,952 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_a321ae427f3731921032.png media/images/validation_200_a321ae427f3731921032.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_009b86ce19fa976b3f5d.png media/images/validation_200_009b86ce19fa976b3f5d.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_200_fe7b08ac2a5afbc11b18.png media/images/validation_200_fe7b08ac2a5afbc11b18.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_1f72d7f053af939e5276.png media/images/validation_210_1f72d7f053af939e5276.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_ade80344c97e522275e6.png media/images/validation_210_ade80344c97e522275e6.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_b7889716d369f8c47db2.png media/images/validation_210_b7889716d369f8c47db2.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_210_30fb41bcb1ce089cb7f8.png media/images/validation_210_30fb41bcb1ce089cb7f8.png +2023-07-12 12:35:46,953 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_f581c132868ccb4db30b.png media/images/validation_220_f581c132868ccb4db30b.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_1b6340fa1d30a762feb0.png media/images/validation_220_1b6340fa1d30a762feb0.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_58119209c10f17b2d40d.png media/images/validation_220_58119209c10f17b2d40d.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_220_7e556c2aaa57f2ce8afe.png media/images/validation_220_7e556c2aaa57f2ce8afe.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_a1e3c8f3087f5e3012a5.png media/images/validation_230_a1e3c8f3087f5e3012a5.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_16d4939d62b11379b933.png media/images/validation_230_16d4939d62b11379b933.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_9dcda9052cdd403f3897.png media/images/validation_230_9dcda9052cdd403f3897.png +2023-07-12 12:35:46,954 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_230_d631c6d270c38f7631a9.png media/images/validation_230_d631c6d270c38f7631a9.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_5cfbeaed31a9337bdbad.png media/images/validation_240_5cfbeaed31a9337bdbad.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_18282123a377a69416e7.png media/images/validation_240_18282123a377a69416e7.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_98861fa9aa94e0bb9466.png media/images/validation_240_98861fa9aa94e0bb9466.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_240_607b3eebed55e6abe830.png media/images/validation_240_607b3eebed55e6abe830.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_96a516914a407fcfc504.png media/images/validation_250_96a516914a407fcfc504.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_7a35da35c250d174b078.png media/images/validation_250_7a35da35c250d174b078.png +2023-07-12 12:35:46,955 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_8833dd1eeb120dd86be3.png media/images/validation_250_8833dd1eeb120dd86be3.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_250_cc24f9252c2efc365a1d.png media/images/validation_250_cc24f9252c2efc365a1d.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_9d432cf0ad1538825247.png media/images/validation_260_9d432cf0ad1538825247.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c5ef4fbb8722ad5da91d.png media/images/validation_260_c5ef4fbb8722ad5da91d.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_4f96826bcb36d428d2d7.png media/images/validation_260_4f96826bcb36d428d2d7.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_260_c2fb4e9073164765d619.png media/images/validation_260_c2fb4e9073164765d619.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_3fe25e6178ac8d39607e.png media/images/validation_270_3fe25e6178ac8d39607e.png +2023-07-12 12:35:46,956 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_630c030e959854fa6506.png media/images/validation_270_630c030e959854fa6506.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_5145710341bd620359f3.png media/images/validation_270_5145710341bd620359f3.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_270_18e1dc0d33fcdedb1fc5.png media/images/validation_270_18e1dc0d33fcdedb1fc5.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_4a948e463af013031d1f.png media/images/validation_280_4a948e463af013031d1f.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_ef3aba43d9e8e71c9284.png media/images/validation_280_ef3aba43d9e8e71c9284.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c25f63eee5ef9870fbd4.png media/images/validation_280_c25f63eee5ef9870fbd4.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_280_c142b41b461357b2de28.png media/images/validation_280_c142b41b461357b2de28.png +2023-07-12 12:35:46,957 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_ff574f4d8f5ce6c5ff1a.png media/images/validation_290_ff574f4d8f5ce6c5ff1a.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_b84e927620cd6b0cb2a8.png media/images/validation_290_b84e927620cd6b0cb2a8.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1c16b044a10bd74e136a.png media/images/validation_290_1c16b044a10bd74e136a.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_290_1b1094343a18b1720c55.png media/images/validation_290_1b1094343a18b1720c55.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_a2de729812e74d6ab884.png media/images/validation_300_a2de729812e74d6ab884.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_872691b49d402d6a737f.png media/images/validation_300_872691b49d402d6a737f.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1dc75f8c8767180b4449.png media/images/validation_300_1dc75f8c8767180b4449.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_300_1a544bb059e5c537308c.png media/images/validation_300_1a544bb059e5c537308c.png +2023-07-12 12:35:46,958 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_619cd7b6e4fcfa89810a.png media/images/validation_310_619cd7b6e4fcfa89810a.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_5a10ea9a10a25e57135c.png media/images/validation_310_5a10ea9a10a25e57135c.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_43c929f6445607c5f8d2.png media/images/validation_310_43c929f6445607c5f8d2.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_310_01b83aa2726efc837f38.png media/images/validation_310_01b83aa2726efc837f38.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_73ad6d9a88337e318469.png media/images/validation_320_73ad6d9a88337e318469.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_a373a58713d055f4d0ec.png media/images/validation_320_a373a58713d055f4d0ec.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_71caeb46293b077a4cd4.png media/images/validation_320_71caeb46293b077a4cd4.png +2023-07-12 12:35:46,959 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_320_1627810819d85c5e49c5.png media/images/validation_320_1627810819d85c5e49c5.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_4559efd7cd62cbc0d7bd.png media/images/validation_330_4559efd7cd62cbc0d7bd.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_fc7d6afb7198af162914.png media/images/validation_330_fc7d6afb7198af162914.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_7fbcfe2f1f50d8f10ad6.png media/images/validation_330_7fbcfe2f1f50d8f10ad6.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_330_103af67d972f2199c1fc.png media/images/validation_330_103af67d972f2199c1fc.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_f1d20714dd011423e767.png media/images/validation_340_f1d20714dd011423e767.png +2023-07-12 12:35:46,960 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_dc0f97f353b7e7ab6dca.png media/images/validation_340_dc0f97f353b7e7ab6dca.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_457d6bff4c2a7ae5f48a.png media/images/validation_340_457d6bff4c2a7ae5f48a.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_340_2a1bc7c87c68196530cf.png media/images/validation_340_2a1bc7c87c68196530cf.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_22aac1153c97ec2693c5.png media/images/validation_350_22aac1153c97ec2693c5.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_e283ca771222ebaf7b13.png media/images/validation_350_e283ca771222ebaf7b13.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_bb4784ba59026990690d.png media/images/validation_350_bb4784ba59026990690d.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_350_1c59f118bd1aa7b351ef.png media/images/validation_350_1c59f118bd1aa7b351ef.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_26e2ebd54ecac5225fde.png media/images/validation_360_26e2ebd54ecac5225fde.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_d5ce8065e09a898bbb73.png media/images/validation_360_d5ce8065e09a898bbb73.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_1483311cc0a22899fd84.png media/images/validation_360_1483311cc0a22899fd84.png +2023-07-12 12:35:46,961 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_360_09d003a6833146016be6.png media/images/validation_360_09d003a6833146016be6.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_66cc1fe983e4ffdf1c5b.png media/images/validation_370_66cc1fe983e4ffdf1c5b.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_fa56e2608c00ceaca35e.png media/images/validation_370_fa56e2608c00ceaca35e.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_0b641787504f5e185899.png media/images/validation_370_0b641787504f5e185899.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_370_8ad08c11ab8ad22829d8.png media/images/validation_370_8ad08c11ab8ad22829d8.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_8126eeaacd234695183b.png media/images/validation_380_8126eeaacd234695183b.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_e3c3849a148163ace395.png media/images/validation_380_e3c3849a148163ace395.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_150783a4c31c17364129.png media/images/validation_380_150783a4c31c17364129.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_380_cf0b270e00fd03447bc3.png media/images/validation_380_cf0b270e00fd03447bc3.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_4f13ece0060ef305e7c1.png media/images/validation_390_4f13ece0060ef305e7c1.png +2023-07-12 12:35:46,962 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_cece94bd69f475704bfa.png media/images/validation_390_cece94bd69f475704bfa.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_b80214c7e7fafa67c5b0.png media/images/validation_390_b80214c7e7fafa67c5b0.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_390_dea73b102aef933ebfb7.png media/images/validation_390_dea73b102aef933ebfb7.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_b8bc388d0a6aa1ae82dc.png media/images/validation_400_b8bc388d0a6aa1ae82dc.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_301c182c9a0503b9c552.png media/images/validation_400_301c182c9a0503b9c552.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_d5487564dfd56ce35ceb.png media/images/validation_400_d5487564dfd56ce35ceb.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_400_5cbb16c19b007f3c49d3.png media/images/validation_400_5cbb16c19b007f3c49d3.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_36160e0e12a477c07ef9.png media/images/validation_410_36160e0e12a477c07ef9.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_975e274aed0125eac405.png media/images/validation_410_975e274aed0125eac405.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_c88421f1a33201c27e7f.png media/images/validation_410_c88421f1a33201c27e7f.png +2023-07-12 12:35:46,963 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_410_3032692fd49e0f4ffa6a.png media/images/validation_410_3032692fd49e0f4ffa6a.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_2a287e605f741cc4ba84.png media/images/validation_420_2a287e605f741cc4ba84.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_f17cfd3caf204c385f2c.png media/images/validation_420_f17cfd3caf204c385f2c.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_b5f6b2004b8fac7ef053.png media/images/validation_420_b5f6b2004b8fac7ef053.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_420_e80fbc111f2744e80e0e.png media/images/validation_420_e80fbc111f2744e80e0e.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_582c2e8324c4a214eee2.png media/images/validation_430_582c2e8324c4a214eee2.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_31a8bb8062237a4e5e03.png media/images/validation_430_31a8bb8062237a4e5e03.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_cc52ea02c073b4adb746.png media/images/validation_430_cc52ea02c073b4adb746.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_430_466a9d653c954a92a2d2.png media/images/validation_430_466a9d653c954a92a2d2.png +2023-07-12 12:35:46,964 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_996fc8410c2def6b1ad3.png media/images/validation_440_996fc8410c2def6b1ad3.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_b7c3984877608e60dcc5.png media/images/validation_440_b7c3984877608e60dcc5.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_37feadf7c6d79ab08bb8.png media/images/validation_440_37feadf7c6d79ab08bb8.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_440_850de8645d4e0ec1b327.png media/images/validation_440_850de8645d4e0ec1b327.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_62e3070332b5e7b50721.png media/images/validation_450_62e3070332b5e7b50721.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_129489423a36c578b83b.png media/images/validation_450_129489423a36c578b83b.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_0be657c2999bb3f2f0e8.png media/images/validation_450_0be657c2999bb3f2f0e8.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_450_75c8cb1b49d83b55316d.png media/images/validation_450_75c8cb1b49d83b55316d.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_a6c4a260b23627355952.png media/images/validation_460_a6c4a260b23627355952.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_69bab95618e258fac08d.png media/images/validation_460_69bab95618e258fac08d.png +2023-07-12 12:35:46,965 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_de7722337cbd652fed8a.png media/images/validation_460_de7722337cbd652fed8a.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_460_387823557d6d5cb784bb.png media/images/validation_460_387823557d6d5cb784bb.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f98b8202065d26c1ef2f.png media/images/validation_470_f98b8202065d26c1ef2f.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_ae66beb9ba6213202415.png media/images/validation_470_ae66beb9ba6213202415.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_729f9b8d68182ccc9b8c.png media/images/validation_470_729f9b8d68182ccc9b8c.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_470_f3357ea3af4ad6b0deea.png media/images/validation_470_f3357ea3af4ad6b0deea.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_57793f0e6246b307afce.png media/images/validation_480_57793f0e6246b307afce.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_36f12af806c3f1d50a42.png media/images/validation_480_36f12af806c3f1d50a42.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_f389fd00788c87466ea2.png media/images/validation_480_f389fd00788c87466ea2.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_480_11e4b59458340be3e02c.png media/images/validation_480_11e4b59458340be3e02c.png +2023-07-12 12:35:46,966 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_e1df59c538a185bbc3ab.png media/images/validation_490_e1df59c538a185bbc3ab.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_bb00c284bf371c7cee01.png media/images/validation_490_bb00c284bf371c7cee01.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_68abe5bccc0897c1b607.png media/images/validation_490_68abe5bccc0897c1b607.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_490_47a8d03f0cc699178621.png media/images/validation_490_47a8d03f0cc699178621.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_03a01949302184444647.png media/images/validation_500_03a01949302184444647.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d74931567eda76fb139.png media/images/validation_500_0d74931567eda76fb139.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_0d16f7c02c5cc655df6a.png media/images/validation_500_0d16f7c02c5cc655df6a.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_500_89f5a8910a8c98de8cb9.png media/images/validation_500_89f5a8910a8c98de8cb9.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_3234686b4d18835995c9.png media/images/validation_510_3234686b4d18835995c9.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_da96661dc21e252cae64.png media/images/validation_510_da96661dc21e252cae64.png +2023-07-12 12:35:46,967 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_34fac9bf4212cdd6d437.png media/images/validation_510_34fac9bf4212cdd6d437.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_510_e3e834df071a8745bc59.png media/images/validation_510_e3e834df071a8745bc59.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_61f1eebc64efc60d8ae6.png media/images/validation_520_61f1eebc64efc60d8ae6.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_8d7aa871f805bb466b02.png media/images/validation_520_8d7aa871f805bb466b02.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_95ea6a7d25f4eeb9af52.png media/images/validation_520_95ea6a7d25f4eeb9af52.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_520_c82ca30fe9549b880c53.png media/images/validation_520_c82ca30fe9549b880c53.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_d9608828cb52fcf1ad77.png media/images/validation_530_d9608828cb52fcf1ad77.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_a5b58d688f92b4fc1442.png media/images/validation_530_a5b58d688f92b4fc1442.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_001908c5ae0df54582c9.png media/images/validation_530_001908c5ae0df54582c9.png +2023-07-12 12:35:46,968 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_530_9ebd350e55790857624d.png media/images/validation_530_9ebd350e55790857624d.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_978f41b570f2d5fd48e5.png media/images/validation_540_978f41b570f2d5fd48e5.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_68ef6d897da63a408d01.png media/images/validation_540_68ef6d897da63a408d01.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_37a3b637808816ae2a1f.png media/images/validation_540_37a3b637808816ae2a1f.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_540_fc4fe59f596766ac60b6.png media/images/validation_540_fc4fe59f596766ac60b6.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_90a56ad8ada0c9c8274d.png media/images/validation_550_90a56ad8ada0c9c8274d.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_116ac864f0f13e57d81b.png media/images/validation_550_116ac864f0f13e57d81b.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_55055c73d3e5ab4513f3.png media/images/validation_550_55055c73d3e5ab4513f3.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_550_402084574a959b5e4266.png media/images/validation_550_402084574a959b5e4266.png +2023-07-12 12:35:46,969 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_607b86b7a89c36fdadfe.png media/images/validation_560_607b86b7a89c36fdadfe.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a72665b3636d99570f6e.png media/images/validation_560_a72665b3636d99570f6e.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_a54da6547e3af36c2510.png media/images/validation_560_a54da6547e3af36c2510.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_560_d160b3bcabae5e649acf.png media/images/validation_560_d160b3bcabae5e649acf.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_6d9eeb6998ef5daf6dc1.png media/images/validation_570_6d9eeb6998ef5daf6dc1.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_eb681e116684d158f76a.png media/images/validation_570_eb681e116684d158f76a.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_e92644ed701dc0b0faa2.png media/images/validation_570_e92644ed701dc0b0faa2.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_570_9c89c3127ee7dbbf4a2a.png media/images/validation_570_9c89c3127ee7dbbf4a2a.png +2023-07-12 12:35:46,970 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b26c685674fa1336fbab.png media/images/validation_580_b26c685674fa1336fbab.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_fd8f16b41072ce3bceb4.png media/images/validation_580_fd8f16b41072ce3bceb4.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_b60bdd2ea4c17924c6d0.png media/images/validation_580_b60bdd2ea4c17924c6d0.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_580_d75cf57bdd0e29b1f7a8.png media/images/validation_580_d75cf57bdd0e29b1f7a8.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_a98fc70c70656fdfe63b.png media/images/validation_590_a98fc70c70656fdfe63b.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_08863b59853c2669a10c.png media/images/validation_590_08863b59853c2669a10c.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_e05be95e710336b098d2.png media/images/validation_590_e05be95e710336b098d2.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_590_06bde4208de6769e990b.png media/images/validation_590_06bde4208de6769e990b.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_e5844c86288b2606b645.png media/images/validation_600_e5844c86288b2606b645.png +2023-07-12 12:35:46,971 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_07de5acd372977760b4c.png media/images/validation_600_07de5acd372977760b4c.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_bbb685abfd63c223246e.png media/images/validation_600_bbb685abfd63c223246e.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_600_6b5551cfcbdcbf8c81e5.png media/images/validation_600_6b5551cfcbdcbf8c81e5.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_1d981ecc323c30e20b81.png media/images/validation_610_1d981ecc323c30e20b81.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_b9cc12355deddaad9779.png media/images/validation_610_b9cc12355deddaad9779.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_134e6d2a749fcacd43b7.png media/images/validation_610_134e6d2a749fcacd43b7.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_610_92f0f0898c20a134b3c3.png media/images/validation_610_92f0f0898c20a134b3c3.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_77093872b2a62f3184c0.png media/images/validation_620_77093872b2a62f3184c0.png +2023-07-12 12:35:46,972 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_f72e5adad3a7e2192d4a.png media/images/validation_620_f72e5adad3a7e2192d4a.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ed7326150a0c47d8bd9c.png media/images/validation_620_ed7326150a0c47d8bd9c.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_620_ad1a36db317705c6c410.png media/images/validation_620_ad1a36db317705c6c410.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_4f603f3934c8321cc8fb.png media/images/validation_630_4f603f3934c8321cc8fb.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_79440b6bdd999b2e130a.png media/images/validation_630_79440b6bdd999b2e130a.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_da9c8b5a7202534ddcc5.png media/images/validation_630_da9c8b5a7202534ddcc5.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_630_bdab7dddcf554aad01f9.png media/images/validation_630_bdab7dddcf554aad01f9.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_49779e88089791939e26.png media/images/validation_640_49779e88089791939e26.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_b64ccfa16d3aa44a2151.png media/images/validation_640_b64ccfa16d3aa44a2151.png +2023-07-12 12:35:46,973 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_56616402a26a74e9ca6a.png media/images/validation_640_56616402a26a74e9ca6a.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_640_c1d1fb2c049800d964ec.png media/images/validation_640_c1d1fb2c049800d964ec.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_7416b90b79fecff5e4ca.png media/images/validation_650_7416b90b79fecff5e4ca.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_606f3fe637934aa6a15b.png media/images/validation_650_606f3fe637934aa6a15b.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_2b23cf43dc0d03af66c6.png media/images/validation_650_2b23cf43dc0d03af66c6.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_650_774f3994c62fdf69ce65.png media/images/validation_650_774f3994c62fdf69ce65.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_aca4a003a8d6b3b9f87b.png media/images/validation_660_aca4a003a8d6b3b9f87b.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_4155d6ba117c33d5628b.png media/images/validation_660_4155d6ba117c33d5628b.png +2023-07-12 12:35:46,974 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_65494724a21e7695e782.png media/images/validation_660_65494724a21e7695e782.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_660_3806edfc6615feb4c502.png media/images/validation_660_3806edfc6615feb4c502.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_00b528cb36f02641e0c4.png media/images/validation_670_00b528cb36f02641e0c4.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_4b3b71fb783337ee170d.png media/images/validation_670_4b3b71fb783337ee170d.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_a54a1c21b8282d62f8f9.png media/images/validation_670_a54a1c21b8282d62f8f9.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_670_38a8a2cd9ee27689f5ea.png media/images/validation_670_38a8a2cd9ee27689f5ea.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_7d58abcefc10987ef6e0.png media/images/validation_680_7d58abcefc10987ef6e0.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_96b5253932542ff8e0ec.png media/images/validation_680_96b5253932542ff8e0ec.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_5420d62f64c76dfb3aaf.png media/images/validation_680_5420d62f64c76dfb3aaf.png +2023-07-12 12:35:46,975 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_680_ee4486348a50df1eb1b0.png media/images/validation_680_ee4486348a50df1eb1b0.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_1ff9e60aee819ba10a7e.png media/images/validation_690_1ff9e60aee819ba10a7e.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_8220342bb1673b65f66a.png media/images/validation_690_8220342bb1673b65f66a.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_9749b628dfc772a6cc47.png media/images/validation_690_9749b628dfc772a6cc47.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_690_dd5fdf92beda60fe8a12.png media/images/validation_690_dd5fdf92beda60fe8a12.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_002747928010272ad4e0.png media/images/validation_700_002747928010272ad4e0.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_f85b220903af16d48ab4.png media/images/validation_700_f85b220903af16d48ab4.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_ccf6b7bd526cf61a21d8.png media/images/validation_700_ccf6b7bd526cf61a21d8.png +2023-07-12 12:35:46,976 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_700_11469b6c8fd1d388b8bd.png media/images/validation_700_11469b6c8fd1d388b8bd.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_f8ad001e252a38c302da.png media/images/validation_710_f8ad001e252a38c302da.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_e29ac85551bb1427b956.png media/images/validation_710_e29ac85551bb1427b956.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_4e2183818f3e746eee9b.png media/images/validation_710_4e2183818f3e746eee9b.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_710_42e1f742968073010893.png media/images/validation_710_42e1f742968073010893.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_0ecdc87dea1892df4eaf.png media/images/validation_720_0ecdc87dea1892df4eaf.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_9c940486a7ed24710766.png media/images/validation_720_9c940486a7ed24710766.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_a812b1f440f7f52d6145.png media/images/validation_720_a812b1f440f7f52d6145.png +2023-07-12 12:35:46,977 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_720_dc11c1be8869686a04d9.png media/images/validation_720_dc11c1be8869686a04d9.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_61523736602b93efd4f1.png media/images/validation_730_61523736602b93efd4f1.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_b3042d96cd6c0051282b.png media/images/validation_730_b3042d96cd6c0051282b.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_092236683fae816d1e06.png media/images/validation_730_092236683fae816d1e06.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_730_0d352eba665388c33ae3.png media/images/validation_730_0d352eba665388c33ae3.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_4ca01dfc0ecd320c19a8.png media/images/validation_740_4ca01dfc0ecd320c19a8.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_472a400ca408d21e03bd.png media/images/validation_740_472a400ca408d21e03bd.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_d97239257eb1f6f99304.png media/images/validation_740_d97239257eb1f6f99304.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_740_46e887ae6b4e77b77e34.png media/images/validation_740_46e887ae6b4e77b77e34.png +2023-07-12 12:35:46,978 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_6a4042dd0f145a5a90ed.png media/images/validation_750_6a4042dd0f145a5a90ed.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_d0782df35bd2b2c3cded.png media/images/validation_750_d0782df35bd2b2c3cded.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_f007accf327156196715.png media/images/validation_750_f007accf327156196715.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_750_952512d46b1737ec3172.png media/images/validation_750_952512d46b1737ec3172.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_34b823a11a0cf4013af4.png media/images/validation_760_34b823a11a0cf4013af4.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_af5422b14a97cc645625.png media/images/validation_760_af5422b14a97cc645625.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_95d5c0a5c24789a3a475.png media/images/validation_760_95d5c0a5c24789a3a475.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_760_00dc8fdce08e1ae13772.png media/images/validation_760_00dc8fdce08e1ae13772.png +2023-07-12 12:35:46,979 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_df68bb8ae7cb3ad86921.png media/images/validation_770_df68bb8ae7cb3ad86921.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png media/images/validation_770_5fa4d66ae0a6f8b3ea3d.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_cba209758498588b8517.png media/images/validation_770_cba209758498588b8517.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_770_217409d7c0158fc1dcec.png media/images/validation_770_217409d7c0158fc1dcec.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_5bc6ddbcab2869abe354.png media/images/validation_780_5bc6ddbcab2869abe354.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cd3d57f7ce32e8a2128.png media/images/validation_780_8cd3d57f7ce32e8a2128.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_8cb1b948d1d2b956831d.png media/images/validation_780_8cb1b948d1d2b956831d.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_780_6240b2c0fb96b2b24cb6.png media/images/validation_780_6240b2c0fb96b2b24cb6.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_f18cdbdebab2bd2197d4.png media/images/validation_790_f18cdbdebab2bd2197d4.png +2023-07-12 12:35:46,980 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_7c032ae6c9675c498efb.png media/images/validation_790_7c032ae6c9675c498efb.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_0e3bece4b24d9f0cff63.png media/images/validation_790_0e3bece4b24d9f0cff63.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_790_2c76661900a84617ca31.png media/images/validation_790_2c76661900a84617ca31.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_a66b82318049cd1e171c.png media/images/validation_800_a66b82318049cd1e171c.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_ab1fea6190aed6bcea40.png media/images/validation_800_ab1fea6190aed6bcea40.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_c3dd46dd047d49388d6c.png media/images/validation_800_c3dd46dd047d49388d6c.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_800_9c63a115f12e85651901.png media/images/validation_800_9c63a115f12e85651901.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_9a62636574ba68d8fc83.png media/images/validation_810_9a62636574ba68d8fc83.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_8d25802cfd68926c4540.png media/images/validation_810_8d25802cfd68926c4540.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_14a8076e3481bc7a8090.png media/images/validation_810_14a8076e3481bc7a8090.png +2023-07-12 12:35:46,981 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_810_3d586403ca68c8dd89ef.png media/images/validation_810_3d586403ca68c8dd89ef.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_9eeeace7edf859719f14.png media/images/validation_820_9eeeace7edf859719f14.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_498f985682f98f0b515d.png media/images/validation_820_498f985682f98f0b515d.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_c37a81321646db9654fb.png media/images/validation_820_c37a81321646db9654fb.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_820_de63bd88f0f3c4502afa.png media/images/validation_820_de63bd88f0f3c4502afa.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_5d081a2a0528a30e8ce2.png media/images/validation_830_5d081a2a0528a30e8ce2.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_0aca2d7047838c676be5.png media/images/validation_830_0aca2d7047838c676be5.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_3bdbbfd9d4a0076d30db.png media/images/validation_830_3bdbbfd9d4a0076d30db.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_830_73577e16f9e832867bc6.png media/images/validation_830_73577e16f9e832867bc6.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_90e1e0a7b7d300559299.png media/images/validation_840_90e1e0a7b7d300559299.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_3cc72b352f6a24b1d731.png media/images/validation_840_3cc72b352f6a24b1d731.png +2023-07-12 12:35:46,982 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_924f6a01834fef3e5df9.png media/images/validation_840_924f6a01834fef3e5df9.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_840_8376c90a668eab0df0f1.png media/images/validation_840_8376c90a668eab0df0f1.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_d8f23be87659ed0cbfb4.png media/images/validation_850_d8f23be87659ed0cbfb4.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20f5209698af0ab0b778.png media/images/validation_850_20f5209698af0ab0b778.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_20c9060ffc3bdad4a934.png media/images/validation_850_20c9060ffc3bdad4a934.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_850_1f72867e944a10d3e520.png media/images/validation_850_1f72867e944a10d3e520.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_97fe22a2e02d107e0669.png media/images/validation_860_97fe22a2e02d107e0669.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_8062bd4106d50ccb836d.png media/images/validation_860_8062bd4106d50ccb836d.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_578c282df31c9a089068.png media/images/validation_860_578c282df31c9a089068.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_860_1ff63456305716e15a0f.png media/images/validation_860_1ff63456305716e15a0f.png +2023-07-12 12:35:46,983 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_b6004993162c340948e4.png media/images/validation_870_b6004993162c340948e4.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_1e7e11e8737344ca3c57.png media/images/validation_870_1e7e11e8737344ca3c57.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_4f7e5756a84f41b58f03.png media/images/validation_870_4f7e5756a84f41b58f03.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_870_674d99e976453deac9a0.png media/images/validation_870_674d99e976453deac9a0.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_c05827ace7407dafe67f.png media/images/validation_880_c05827ace7407dafe67f.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_ddb13591ebb5fc4093e5.png media/images/validation_880_ddb13591ebb5fc4093e5.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_1a93cee3c2fe9238634b.png media/images/validation_880_1a93cee3c2fe9238634b.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_880_8a9698cc7d3b8724fb2b.png media/images/validation_880_8a9698cc7d3b8724fb2b.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_f18e48cd66c93df800cc.png media/images/validation_890_f18e48cd66c93df800cc.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_5f8815ecaca9bc2b8ad7.png media/images/validation_890_5f8815ecaca9bc2b8ad7.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_01b5da4e3887f97a0716.png media/images/validation_890_01b5da4e3887f97a0716.png +2023-07-12 12:35:46,984 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_890_e9f5b2cb5a42a51a191f.png media/images/validation_890_e9f5b2cb5a42a51a191f.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_6956b8efd11196c8a652.png media/images/validation_900_6956b8efd11196c8a652.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_5741bd49ef7880023ce2.png media/images/validation_900_5741bd49ef7880023ce2.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_54eb3287687fb6f3d07f.png media/images/validation_900_54eb3287687fb6f3d07f.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_900_4bf5e195f1228d67ca30.png media/images/validation_900_4bf5e195f1228d67ca30.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_f56c6743f3df48566c7e.png media/images/validation_910_f56c6743f3df48566c7e.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_85b1581a7259fb0cc926.png media/images/validation_910_85b1581a7259fb0cc926.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_4bf762954166a9bf2bce.png media/images/validation_910_4bf762954166a9bf2bce.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_910_2e5e7f759b91d9be266a.png media/images/validation_910_2e5e7f759b91d9be266a.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_3ac6586c86738660d7b9.png media/images/validation_920_3ac6586c86738660d7b9.png +2023-07-12 12:35:46,985 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_7f02fda774e0cc0ec92a.png media/images/validation_920_7f02fda774e0cc0ec92a.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_8544a6b986b0e7758e17.png media/images/validation_920_8544a6b986b0e7758e17.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_920_4cbb72cb2a336b2f4f2b.png media/images/validation_920_4cbb72cb2a336b2f4f2b.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_a2e24d69369ee9545e6c.png media/images/validation_930_a2e24d69369ee9545e6c.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_98329b1eb9c39ec50100.png media/images/validation_930_98329b1eb9c39ec50100.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_14811891692332578df2.png media/images/validation_930_14811891692332578df2.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_930_708543de31a4012af277.png media/images/validation_930_708543de31a4012af277.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_1aa1dd5671e7761dc7f9.png media/images/validation_940_1aa1dd5671e7761dc7f9.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_25baa72fa29a6b87a7e3.png media/images/validation_940_25baa72fa29a6b87a7e3.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_55fb249dfc64d037ba51.png media/images/validation_940_55fb249dfc64d037ba51.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_940_0f4ce489e03b7c6499a4.png media/images/validation_940_0f4ce489e03b7c6499a4.png +2023-07-12 12:35:46,986 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_8e55f5be94ca16a527e8.png media/images/validation_950_8e55f5be94ca16a527e8.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_cedfd2e3a8d728719314.png media/images/validation_950_cedfd2e3a8d728719314.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_b66230be0915df8b41e3.png media/images/validation_950_b66230be0915df8b41e3.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_950_32af62f9d0e790c02608.png media/images/validation_950_32af62f9d0e790c02608.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_a3b1bfcfc8aabde7212f.png media/images/validation_960_a3b1bfcfc8aabde7212f.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_5c7c719c7e9e5e608018.png media/images/validation_960_5c7c719c7e9e5e608018.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_fdf70bc0fef8dac9c1a0.png media/images/validation_960_fdf70bc0fef8dac9c1a0.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_960_67790e930adf85923341.png media/images/validation_960_67790e930adf85923341.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_4600d4c65eb0657bf15d.png media/images/validation_970_4600d4c65eb0657bf15d.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_133f183508d881d8f1f8.png media/images/validation_970_133f183508d881d8f1f8.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_959d30c344a53f583035.png media/images/validation_970_959d30c344a53f583035.png +2023-07-12 12:35:46,987 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_970_783f685672728f7cab19.png media/images/validation_970_783f685672728f7cab19.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_5bce29b811f38d81b7a9.png media/images/validation_980_5bce29b811f38d81b7a9.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4522b255d71fb495f911.png media/images/validation_980_4522b255d71fb495f911.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4a474756239679aa6533.png media/images/validation_980_4a474756239679aa6533.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_980_4e23c9690e64cda40444.png media/images/validation_980_4e23c9690e64cda40444.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_e555318c04df0d898c35.png media/images/validation_990_e555318c04df0d898c35.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_d4658b3a29548774d64f.png media/images/validation_990_d4658b3a29548774d64f.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_dd615cbf6d5a7a8a896d.png media/images/validation_990_dd615cbf6d5a7a8a896d.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_990_952fc214368e10b389b8.png media/images/validation_990_952fc214368e10b389b8.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_1a96794a04b7af41b34e.png media/images/validation_1000_1a96794a04b7af41b34e.png +2023-07-12 12:35:46,988 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_e0021fa68cabb943f49d.png media/images/validation_1000_e0021fa68cabb943f49d.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_87434c7eeda4190bf8d8.png media/images/validation_1000_87434c7eeda4190bf8d8.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1000_bd2b2cab96e55845d6e1.png media/images/validation_1000_bd2b2cab96e55845d6e1.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_813871e148b66856b8a5.png media/images/validation_1010_813871e148b66856b8a5.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_f52e87f356214ba43ded.png media/images/validation_1010_f52e87f356214ba43ded.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_8213e8a29e4df4a0c573.png media/images/validation_1010_8213e8a29e4df4a0c573.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1010_1fac8c596692012a9a71.png media/images/validation_1010_1fac8c596692012a9a71.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f7760d16dbb61cdadc4b.png media/images/validation_1020_f7760d16dbb61cdadc4b.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_bc519090f4c42a857cf5.png media/images/validation_1020_bc519090f4c42a857cf5.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_d02fa8190f5853b0b648.png media/images/validation_1020_d02fa8190f5853b0b648.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1020_f59fa5400ab07ea1eeae.png media/images/validation_1020_f59fa5400ab07ea1eeae.png +2023-07-12 12:35:46,989 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_bfd7171975011562507f.png media/images/validation_1030_bfd7171975011562507f.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_fd568449fa00bd93527b.png media/images/validation_1030_fd568449fa00bd93527b.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_a95db61e150a9ffc4641.png media/images/validation_1030_a95db61e150a9ffc4641.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1030_3856c65003790d9a56ab.png media/images/validation_1030_3856c65003790d9a56ab.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_15446d27b696fee5751e.png media/images/validation_1040_15446d27b696fee5751e.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_9a1046e9395d322b203b.png media/images/validation_1040_9a1046e9395d322b203b.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_e6fd436066e140c8ca6d.png media/images/validation_1040_e6fd436066e140c8ca6d.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1040_341a0c23b11ea49f2577.png media/images/validation_1040_341a0c23b11ea49f2577.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_55de70da5261f9290e40.png media/images/validation_1050_55de70da5261f9290e40.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_f35c05f1d08c6308ff14.png media/images/validation_1050_f35c05f1d08c6308ff14.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_9bfabce8c467677fe54b.png media/images/validation_1050_9bfabce8c467677fe54b.png +2023-07-12 12:35:46,990 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1050_1daa5fdb77023a407299.png media/images/validation_1050_1daa5fdb77023a407299.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_197022e2780b05209f12.png media/images/validation_1060_197022e2780b05209f12.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_06ba5b400f585cfe22b9.png media/images/validation_1060_06ba5b400f585cfe22b9.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_7be7ad3a7a700e0a150c.png media/images/validation_1060_7be7ad3a7a700e0a150c.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1060_23fba533f1fd02e857f8.png media/images/validation_1060_23fba533f1fd02e857f8.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_32f9f6540dbdc7c48f81.png media/images/validation_1070_32f9f6540dbdc7c48f81.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_68b196a64c6e776ccd9d.png media/images/validation_1070_68b196a64c6e776ccd9d.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_bd0d7850e520c4cb0100.png media/images/validation_1070_bd0d7850e520c4cb0100.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1070_a070795b58d5c596307b.png media/images/validation_1070_a070795b58d5c596307b.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_1f6acd9fcf45ba3ea19f.png media/images/validation_1080_1f6acd9fcf45ba3ea19f.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_4817a2b823e0b176afd7.png media/images/validation_1080_4817a2b823e0b176afd7.png +2023-07-12 12:35:46,991 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_66da8376f918e5f31cac.png media/images/validation_1080_66da8376f918e5f31cac.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1080_6b2cf11ab85982d34290.png media/images/validation_1080_6b2cf11ab85982d34290.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b8740b576006b9ee3546.png media/images/validation_1090_b8740b576006b9ee3546.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_76a3173208a51ee8c312.png media/images/validation_1090_76a3173208a51ee8c312.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_057775edd61cac3840c0.png media/images/validation_1090_057775edd61cac3840c0.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1090_b585e13abb070740f049.png media/images/validation_1090_b585e13abb070740f049.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_b9ea83bc203cb55d812b.png media/images/validation_1100_b9ea83bc203cb55d812b.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_ec943a3b531d5c037829.png media/images/validation_1100_ec943a3b531d5c037829.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_61651029309c609a7521.png media/images/validation_1100_61651029309c609a7521.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1100_1ad4a61cbb8909fcb78e.png media/images/validation_1100_1ad4a61cbb8909fcb78e.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_28d2a849740383813cb6.png media/images/validation_1110_28d2a849740383813cb6.png +2023-07-12 12:35:46,992 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_b889d46b029a022ecfc9.png media/images/validation_1110_b889d46b029a022ecfc9.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_54b9949ee470b53f0543.png media/images/validation_1110_54b9949ee470b53f0543.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1110_eb513f39f1c260344506.png media/images/validation_1110_eb513f39f1c260344506.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_3bb3ea14d134cd57deb1.png media/images/validation_1120_3bb3ea14d134cd57deb1.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_2c6f306607bc7b9bfd92.png media/images/validation_1120_2c6f306607bc7b9bfd92.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_8c2839a1d3140c3a7dfe.png media/images/validation_1120_8c2839a1d3140c3a7dfe.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1120_fd6b36b89f1e7fd2e78a.png media/images/validation_1120_fd6b36b89f1e7fd2e78a.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_93e81f1f307564b83df8.png media/images/validation_1130_93e81f1f307564b83df8.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_331aea85c432690f9c0e.png media/images/validation_1130_331aea85c432690f9c0e.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_44a8006c0888f7a9b714.png media/images/validation_1130_44a8006c0888f7a9b714.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1130_50f80b0ec1f9f7e17923.png media/images/validation_1130_50f80b0ec1f9f7e17923.png +2023-07-12 12:35:46,993 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_1961fdba2d4312886e8c.png media/images/validation_1140_1961fdba2d4312886e8c.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_49f1b6dda25d69bd1eec.png media/images/validation_1140_49f1b6dda25d69bd1eec.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_a158acb6a30a6a4fcc09.png media/images/validation_1140_a158acb6a30a6a4fcc09.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1140_469091e483586cec37b1.png media/images/validation_1140_469091e483586cec37b1.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_9d89ef355de68c0d9ddd.png media/images/validation_1150_9d89ef355de68c0d9ddd.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_bc32f73444fde5093291.png media/images/validation_1150_bc32f73444fde5093291.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_c9a41cef5442cfbcf575.png media/images/validation_1150_c9a41cef5442cfbcf575.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1150_4646158f3cc2ef18d62e.png media/images/validation_1150_4646158f3cc2ef18d62e.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_a90fd9580e1c1e24f448.png media/images/validation_1160_a90fd9580e1c1e24f448.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6b84704eb15e35e5f9a9.png media/images/validation_1160_6b84704eb15e35e5f9a9.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_6e5ebba16091d67e0c49.png media/images/validation_1160_6e5ebba16091d67e0c49.png +2023-07-12 12:35:46,994 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1160_9a7e5eb802701258688c.png media/images/validation_1160_9a7e5eb802701258688c.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_a451b10b1cddeff35521.png media/images/validation_1170_a451b10b1cddeff35521.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_2bc25132345cfabe28d9.png media/images/validation_1170_2bc25132345cfabe28d9.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_f9f3ae84aa2fd03f0c14.png media/images/validation_1170_f9f3ae84aa2fd03f0c14.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1170_e76ace79b32cf06e3105.png media/images/validation_1170_e76ace79b32cf06e3105.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_d085f51cf12dae1345cd.png media/images/validation_1180_d085f51cf12dae1345cd.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_f8627735c220e9667e1c.png media/images/validation_1180_f8627735c220e9667e1c.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_20948b93a6c00143c135.png media/images/validation_1180_20948b93a6c00143c135.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1180_7b2ab005b9837744864c.png media/images/validation_1180_7b2ab005b9837744864c.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_f7abacf0c65108ebd291.png media/images/validation_1190_f7abacf0c65108ebd291.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_c631ebe8975b90c43163.png media/images/validation_1190_c631ebe8975b90c43163.png +2023-07-12 12:35:46,995 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_138faee5a37f54662918.png media/images/validation_1190_138faee5a37f54662918.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1190_597e7e8766e45fd365d0.png media/images/validation_1190_597e7e8766e45fd365d0.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_707f52b23811c66eb81f.png media/images/validation_1200_707f52b23811c66eb81f.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_095c7719d5d88d3a784e.png media/images/validation_1200_095c7719d5d88d3a784e.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_509f77863e0d740b58c5.png media/images/validation_1200_509f77863e0d740b58c5.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1200_c158e5c7fd160b30b673.png media/images/validation_1200_c158e5c7fd160b30b673.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_1ba924a90889d370d938.png media/images/validation_1210_1ba924a90889d370d938.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_25ef2dfcdcaf9ff09dea.png media/images/validation_1210_25ef2dfcdcaf9ff09dea.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_a311e64f75624628145d.png media/images/validation_1210_a311e64f75624628145d.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1210_75f318a5faeb28d48e50.png media/images/validation_1210_75f318a5faeb28d48e50.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_1015b192327aea438f8a.png media/images/validation_1220_1015b192327aea438f8a.png +2023-07-12 12:35:46,996 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_b37517440f90e43b7657.png media/images/validation_1220_b37517440f90e43b7657.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_65b3c4bdf334afcfec36.png media/images/validation_1220_65b3c4bdf334afcfec36.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1220_624173894a8202fd049b.png media/images/validation_1220_624173894a8202fd049b.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_bed029a53fc783aa1494.png media/images/validation_1230_bed029a53fc783aa1494.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_82f6db57a6a62b37c416.png media/images/validation_1230_82f6db57a6a62b37c416.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_0a392241e2b0e5c56de0.png media/images/validation_1230_0a392241e2b0e5c56de0.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1230_fe7e092e619d5a60940f.png media/images/validation_1230_fe7e092e619d5a60940f.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9038ef38623aaf3b1297.png media/images/validation_1240_9038ef38623aaf3b1297.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_4fdefa2382f1383aa03b.png media/images/validation_1240_4fdefa2382f1383aa03b.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_9b87b6c7ef2509f2091e.png media/images/validation_1240_9b87b6c7ef2509f2091e.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1240_563721340a1b7779ebb1.png media/images/validation_1240_563721340a1b7779ebb1.png +2023-07-12 12:35:46,997 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_a92796d341fe8362362e.png media/images/validation_1250_a92796d341fe8362362e.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_7e96d8ec566f76718aea.png media/images/validation_1250_7e96d8ec566f76718aea.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_0ffe8cb158ff46492db0.png media/images/validation_1250_0ffe8cb158ff46492db0.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1250_4abee66cab5fc4486094.png media/images/validation_1250_4abee66cab5fc4486094.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_e893310e8c740aedacd9.png media/images/validation_1260_e893310e8c740aedacd9.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_3930a2a71b1d2e79ebf6.png media/images/validation_1260_3930a2a71b1d2e79ebf6.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_709f8444e3a90e20584a.png media/images/validation_1260_709f8444e3a90e20584a.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1260_52dd5c4bac63f8392a24.png media/images/validation_1260_52dd5c4bac63f8392a24.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_39451786e3d6986a15e5.png media/images/validation_1270_39451786e3d6986a15e5.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_7507600afa47a7721bc0.png media/images/validation_1270_7507600afa47a7721bc0.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_239d53a80d648e3b5d74.png media/images/validation_1270_239d53a80d648e3b5d74.png +2023-07-12 12:35:46,998 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1270_007f79d4a566d7c33ea0.png media/images/validation_1270_007f79d4a566d7c33ea0.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_91a32c15ebfc912c2201.png media/images/validation_1280_91a32c15ebfc912c2201.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_76c909633b7c82a1c57d.png media/images/validation_1280_76c909633b7c82a1c57d.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_d09f421be121a991d346.png media/images/validation_1280_d09f421be121a991d346.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1280_f520f48ddc3348a6c6bd.png media/images/validation_1280_f520f48ddc3348a6c6bd.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_c67265e3a1dfdfb99387.png media/images/validation_1290_c67265e3a1dfdfb99387.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_e8b496a8b73ca169dce8.png media/images/validation_1290_e8b496a8b73ca169dce8.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_ad7f6abdff5cb1890622.png media/images/validation_1290_ad7f6abdff5cb1890622.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1290_499d8794ae851f2ca7c3.png media/images/validation_1290_499d8794ae851f2ca7c3.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_6d25eb3589d810ed3440.png media/images/validation_1300_6d25eb3589d810ed3440.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_295181c2c8f74bdb7122.png media/images/validation_1300_295181c2c8f74bdb7122.png +2023-07-12 12:35:46,999 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_d49ed1edb101f8e16c80.png media/images/validation_1300_d49ed1edb101f8e16c80.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1300_47d0444bbe383a2f3ed2.png media/images/validation_1300_47d0444bbe383a2f3ed2.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_5c747b5d08a35792803f.png media/images/validation_1310_5c747b5d08a35792803f.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_8d7535a093e8d31d2071.png media/images/validation_1310_8d7535a093e8d31d2071.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_b02cf0aa477e22c8930d.png media/images/validation_1310_b02cf0aa477e22c8930d.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1310_59c1d57a32cc82a9a3ff.png media/images/validation_1310_59c1d57a32cc82a9a3ff.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_e5cb375d6da828cc3e3e.png media/images/validation_1320_e5cb375d6da828cc3e3e.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_064f388777b3203b13c5.png media/images/validation_1320_064f388777b3203b13c5.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_a24af0c79adf71caef03.png media/images/validation_1320_a24af0c79adf71caef03.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1320_9bd0004be4c4a0c4f009.png media/images/validation_1320_9bd0004be4c4a0c4f009.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_b1042365de64e0a942b0.png media/images/validation_1330_b1042365de64e0a942b0.png +2023-07-12 12:35:47,000 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_dbe013ae177630d5a987.png media/images/validation_1330_dbe013ae177630d5a987.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_3bca8acf12b4c46ab3ce.png media/images/validation_1330_3bca8acf12b4c46ab3ce.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1330_473197dd1f22461890d6.png media/images/validation_1330_473197dd1f22461890d6.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_87ea3e96f26f5ebb57af.png media/images/validation_1340_87ea3e96f26f5ebb57af.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ccdaf0fc6a17d6760008.png media/images/validation_1340_ccdaf0fc6a17d6760008.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_ff1f03141cde2a233297.png media/images/validation_1340_ff1f03141cde2a233297.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1340_a1fff07a952b497ecc10.png media/images/validation_1340_a1fff07a952b497ecc10.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_f98823b83c4cee5e89ac.png media/images/validation_1350_f98823b83c4cee5e89ac.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8b4cb7fa6ec33b1868bc.png media/images/validation_1350_8b4cb7fa6ec33b1868bc.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_8fdfffa2228e3683ab99.png media/images/validation_1350_8fdfffa2228e3683ab99.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1350_e32477931d4fd98f823b.png media/images/validation_1350_e32477931d4fd98f823b.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_49ed245ee23ca275689e.png media/images/validation_1360_49ed245ee23ca275689e.png +2023-07-12 12:35:47,001 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_7925401a6a5e9671ca9b.png media/images/validation_1360_7925401a6a5e9671ca9b.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_e9a621d16e71d8a70ce6.png media/images/validation_1360_e9a621d16e71d8a70ce6.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1360_fea5b00a54b6cde2ffad.png media/images/validation_1360_fea5b00a54b6cde2ffad.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_482ad8df2ce841450721.png media/images/validation_1370_482ad8df2ce841450721.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_4e545b1d5aa48bae02e7.png media/images/validation_1370_4e545b1d5aa48bae02e7.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_d6a886a91a9a1c97ff8e.png media/images/validation_1370_d6a886a91a9a1c97ff8e.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1370_e377b1a6785b8362f8b8.png media/images/validation_1370_e377b1a6785b8362f8b8.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_ba0b7fdc45db85580b1b.png media/images/validation_1380_ba0b7fdc45db85580b1b.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_d46d1a613eb099818463.png media/images/validation_1380_d46d1a613eb099818463.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_18b8edd8e82489c2e700.png media/images/validation_1380_18b8edd8e82489c2e700.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1380_b7ab393ffe13b943540d.png media/images/validation_1380_b7ab393ffe13b943540d.png +2023-07-12 12:35:47,002 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_2048ef7db009bfd7c05d.png media/images/validation_1390_2048ef7db009bfd7c05d.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_02fa9f2ed8ff12c99aa7.png media/images/validation_1390_02fa9f2ed8ff12c99aa7.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_fb6772a80e5d23777bc1.png media/images/validation_1390_fb6772a80e5d23777bc1.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1390_506d872797968a090e00.png media/images/validation_1390_506d872797968a090e00.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_66aa1d0173b44509cd1d.png media/images/validation_1400_66aa1d0173b44509cd1d.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_e84af5fa7376533300be.png media/images/validation_1400_e84af5fa7376533300be.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_f29eae3956b8b680487b.png media/images/validation_1400_f29eae3956b8b680487b.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1400_d020640095c66cf43293.png media/images/validation_1400_d020640095c66cf43293.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_89c2655d6ea12bc678a5.png media/images/validation_1410_89c2655d6ea12bc678a5.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_754b2c1e14f253b4c743.png media/images/validation_1410_754b2c1e14f253b4c743.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_0de4ce377ce0650003a1.png media/images/validation_1410_0de4ce377ce0650003a1.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1410_57eaa364b09be7b1c708.png media/images/validation_1410_57eaa364b09be7b1c708.png +2023-07-12 12:35:47,003 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_65621df76009498b6856.png media/images/validation_1420_65621df76009498b6856.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_82f075e0330d5c29720e.png media/images/validation_1420_82f075e0330d5c29720e.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_8760dbb0b29df7f09a82.png media/images/validation_1420_8760dbb0b29df7f09a82.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1420_51314bed48a62a63179f.png media/images/validation_1420_51314bed48a62a63179f.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_a5e480b9cae1eda40b25.png media/images/validation_1430_a5e480b9cae1eda40b25.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_6fd754942b22d95c6796.png media/images/validation_1430_6fd754942b22d95c6796.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_bb8a00b843066df3ae5d.png media/images/validation_1430_bb8a00b843066df3ae5d.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1430_acb50883d3668759985a.png media/images/validation_1430_acb50883d3668759985a.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_ff68423dd6be58cda77c.png media/images/validation_1440_ff68423dd6be58cda77c.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_7456d4bbc7e1692f0395.png media/images/validation_1440_7456d4bbc7e1692f0395.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_2d1c628ad6cab3a92695.png media/images/validation_1440_2d1c628ad6cab3a92695.png +2023-07-12 12:35:47,004 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1440_84ea7cea928542cee5c3.png media/images/validation_1440_84ea7cea928542cee5c3.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_6eeb827ca7d660925ab1.png media/images/validation_1450_6eeb827ca7d660925ab1.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_1c96b81934ff3d0e0e7a.png media/images/validation_1450_1c96b81934ff3d0e0e7a.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_f51ce3f90deb8ec9156b.png media/images/validation_1450_f51ce3f90deb8ec9156b.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1450_4fbcb05acbe8b16ae524.png media/images/validation_1450_4fbcb05acbe8b16ae524.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_fc2c6a80029568453397.png media/images/validation_1460_fc2c6a80029568453397.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_e0e86f6d8760c4a8cb7e.png media/images/validation_1460_e0e86f6d8760c4a8cb7e.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_bafecf6d5c7a9080d983.png media/images/validation_1460_bafecf6d5c7a9080d983.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1460_688928104b18c14855ba.png media/images/validation_1460_688928104b18c14855ba.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_70617e686456e8dfd42a.png media/images/validation_1470_70617e686456e8dfd42a.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_724c86f275815432b7e4.png media/images/validation_1470_724c86f275815432b7e4.png +2023-07-12 12:35:47,005 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_841f2b0f1c0c1e17dd9d.png media/images/validation_1470_841f2b0f1c0c1e17dd9d.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1470_d752e1cd90b5707c0547.png media/images/validation_1470_d752e1cd90b5707c0547.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_a4d38678421736956450.png media/images/validation_1480_a4d38678421736956450.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_22921140211d309efc15.png media/images/validation_1480_22921140211d309efc15.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_577a6eadd1895fbf8edd.png media/images/validation_1480_577a6eadd1895fbf8edd.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1480_17a3ddb8f942d0a4a42c.png media/images/validation_1480_17a3ddb8f942d0a4a42c.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_0084cd73f87c0d0b983c.png media/images/validation_1490_0084cd73f87c0d0b983c.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_5443848005d4e6d0afaa.png media/images/validation_1490_5443848005d4e6d0afaa.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_4b4b982230746456caa0.png media/images/validation_1490_4b4b982230746456caa0.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1490_09e2e8132566f85bd196.png media/images/validation_1490_09e2e8132566f85bd196.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_c6e34c37f93334a75319.png media/images/validation_1500_c6e34c37f93334a75319.png +2023-07-12 12:35:47,006 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_de9f5fe5746030c6a74f.png media/images/validation_1500_de9f5fe5746030c6a74f.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_cfb49806a7152804aa8b.png media/images/validation_1500_cfb49806a7152804aa8b.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1500_9a03a9d55b0a9f41b234.png media/images/validation_1500_9a03a9d55b0a9f41b234.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_be748917843477368009.png media/images/validation_1510_be748917843477368009.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_ec02c5d5a2c3f65267f6.png media/images/validation_1510_ec02c5d5a2c3f65267f6.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_d4a2e70a118507b473e4.png media/images/validation_1510_d4a2e70a118507b473e4.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1510_11ba5556c102236aa637.png media/images/validation_1510_11ba5556c102236aa637.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_3d2b9accd30d9956baeb.png media/images/validation_1520_3d2b9accd30d9956baeb.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_634e550898bfbdb9ac9f.png media/images/validation_1520_634e550898bfbdb9ac9f.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_790fbc2ddb08d6dd0d88.png media/images/validation_1520_790fbc2ddb08d6dd0d88.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1520_5140a0409a56d2f968a2.png media/images/validation_1520_5140a0409a56d2f968a2.png +2023-07-12 12:35:47,007 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_c983d04e3b22e1d9100a.png media/images/validation_1530_c983d04e3b22e1d9100a.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_91df6d540afc90aa229c.png media/images/validation_1530_91df6d540afc90aa229c.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_7e2daf917c565f387c73.png media/images/validation_1530_7e2daf917c565f387c73.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1530_6550992879b720b36137.png media/images/validation_1530_6550992879b720b36137.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_5f4cc7e816317d29edbf.png media/images/validation_1540_5f4cc7e816317d29edbf.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_a3ac4868653991969a01.png media/images/validation_1540_a3ac4868653991969a01.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_d7dea15105ec789e1d21.png media/images/validation_1540_d7dea15105ec789e1d21.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1540_cd95bda74fc7ee9c42fd.png media/images/validation_1540_cd95bda74fc7ee9c42fd.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_011beff0267deaa0300b.png media/images/validation_1550_011beff0267deaa0300b.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_8efbafcfdf5d8df07263.png media/images/validation_1550_8efbafcfdf5d8df07263.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_1b7f50867d7f974e4323.png media/images/validation_1550_1b7f50867d7f974e4323.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1550_bd10f02e7a1744e7844e.png media/images/validation_1550_bd10f02e7a1744e7844e.png +2023-07-12 12:35:47,008 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_b236feda2a099cb72a65.png media/images/validation_1560_b236feda2a099cb72a65.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_503b3c4a7c702e3bde04.png media/images/validation_1560_503b3c4a7c702e3bde04.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_fb2e82a82bfa5264522f.png media/images/validation_1560_fb2e82a82bfa5264522f.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1560_137a9e1a20d097ef6ead.png media/images/validation_1560_137a9e1a20d097ef6ead.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_fb05536233b9d9384d99.png media/images/validation_1570_fb05536233b9d9384d99.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_8b73b1d4908d57e0f731.png media/images/validation_1570_8b73b1d4908d57e0f731.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_c996b3e0e81ceb767ce9.png media/images/validation_1570_c996b3e0e81ceb767ce9.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1570_cdd5f325de677a2f831c.png media/images/validation_1570_cdd5f325de677a2f831c.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_934a30ce9258abe3af2d.png media/images/validation_1580_934a30ce9258abe3af2d.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_d7d4dbe10d69395e40ca.png media/images/validation_1580_d7d4dbe10d69395e40ca.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_37f9cd55db6f0502d41d.png media/images/validation_1580_37f9cd55db6f0502d41d.png +2023-07-12 12:35:47,009 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1580_cb784df1b13ebac5484a.png media/images/validation_1580_cb784df1b13ebac5484a.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_d7d78a7a562011ba37c0.png media/images/validation_1590_d7d78a7a562011ba37c0.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_bada9f8f4e29cb81aa9e.png media/images/validation_1590_bada9f8f4e29cb81aa9e.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_da4a4411337b18959a1d.png media/images/validation_1590_da4a4411337b18959a1d.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1590_351faecc0b8489584c40.png media/images/validation_1590_351faecc0b8489584c40.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_a9454bcbee87eab5aea9.png media/images/validation_1600_a9454bcbee87eab5aea9.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_9092d43f6e1c508c45c8.png media/images/validation_1600_9092d43f6e1c508c45c8.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_214ff49c4cb987612b43.png media/images/validation_1600_214ff49c4cb987612b43.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1600_aeea6b2da2c3bf58c7c3.png media/images/validation_1600_aeea6b2da2c3bf58c7c3.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_1733cdf33e35398110c3.png media/images/validation_1610_1733cdf33e35398110c3.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_b6d946329d7752479070.png media/images/validation_1610_b6d946329d7752479070.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_f89b822e56af0bee16c1.png media/images/validation_1610_f89b822e56af0bee16c1.png +2023-07-12 12:35:47,010 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1610_49a77c36b783c7c3533f.png media/images/validation_1610_49a77c36b783c7c3533f.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5f0c6d880a0a69fc8df8.png media/images/validation_1620_5f0c6d880a0a69fc8df8.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_cab2ce99009b26517a5a.png media/images/validation_1620_cab2ce99009b26517a5a.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_5e34d10c6734761c103e.png media/images/validation_1620_5e34d10c6734761c103e.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1620_0772971e314e9d07f59b.png media/images/validation_1620_0772971e314e9d07f59b.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_72968a408eafe4615fe4.png media/images/validation_1630_72968a408eafe4615fe4.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_de760783b75bdc640c99.png media/images/validation_1630_de760783b75bdc640c99.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_bcdd70acdf0c52445fdc.png media/images/validation_1630_bcdd70acdf0c52445fdc.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1630_661f798cc99bf53f5fc2.png media/images/validation_1630_661f798cc99bf53f5fc2.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_a26ecf3db23e2ccac681.png media/images/validation_1640_a26ecf3db23e2ccac681.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_585881b9e0f6d40e0ec3.png media/images/validation_1640_585881b9e0f6d40e0ec3.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_68a5f846f45f697c2ead.png media/images/validation_1640_68a5f846f45f697c2ead.png +2023-07-12 12:35:47,011 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1640_64d777e8b185c0851910.png media/images/validation_1640_64d777e8b185c0851910.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_f24a9e2785b102661151.png media/images/validation_1650_f24a9e2785b102661151.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_dda90c1223ed981d3b1a.png media/images/validation_1650_dda90c1223ed981d3b1a.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_c45fc62adfcc71af96fa.png media/images/validation_1650_c45fc62adfcc71af96fa.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1650_3e49f286e7ccf53fddc7.png media/images/validation_1650_3e49f286e7ccf53fddc7.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_000cc75e10c8a7929ff7.png media/images/validation_1660_000cc75e10c8a7929ff7.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_bcf133d130695d2171a4.png media/images/validation_1660_bcf133d130695d2171a4.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_9e45cdde0ed95786481b.png media/images/validation_1660_9e45cdde0ed95786481b.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1660_d35268c957b3894ad085.png media/images/validation_1660_d35268c957b3894ad085.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_c0361568b66056400dc2.png media/images/validation_1670_c0361568b66056400dc2.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f129804ddaa9a57cd45c.png media/images/validation_1670_f129804ddaa9a57cd45c.png +2023-07-12 12:35:47,012 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_4f3ac5bc86e6378f1a12.png media/images/validation_1670_4f3ac5bc86e6378f1a12.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1670_f237ce678dbe9ec09de4.png media/images/validation_1670_f237ce678dbe9ec09de4.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_731d1b60253f6f4063e4.png media/images/validation_1680_731d1b60253f6f4063e4.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_f040c34685a98d0f3850.png media/images/validation_1680_f040c34685a98d0f3850.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_8d56acd02a2f045be997.png media/images/validation_1680_8d56acd02a2f045be997.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1680_80603d23a803515f19ef.png media/images/validation_1680_80603d23a803515f19ef.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png media/images/validation_1690_1ebfe43d0e1ce6cd34dc.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_7295168e3d00a0e3d439.png media/images/validation_1690_7295168e3d00a0e3d439.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_fbcc4f86c9d83ede1f5d.png media/images/validation_1690_fbcc4f86c9d83ede1f5d.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1690_c1341754ea0507193be5.png media/images/validation_1690_c1341754ea0507193be5.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f5c67f9540d8b5ed5c7c.png media/images/validation_1700_f5c67f9540d8b5ed5c7c.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_f31d3c5e28119395907e.png media/images/validation_1700_f31d3c5e28119395907e.png +2023-07-12 12:35:47,013 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_508c64e8a2ff4f92e95d.png media/images/validation_1700_508c64e8a2ff4f92e95d.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1700_46dc3e4359141341c6d4.png media/images/validation_1700_46dc3e4359141341c6d4.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_bbaae7ee7d09d30b3e1a.png media/images/validation_1710_bbaae7ee7d09d30b3e1a.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_92fe9627890fe0330f34.png media/images/validation_1710_92fe9627890fe0330f34.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_be9cd4b95a81eb492abf.png media/images/validation_1710_be9cd4b95a81eb492abf.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1710_603a92e14eb64a33c047.png media/images/validation_1710_603a92e14eb64a33c047.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_80be20563f1620981e46.png media/images/validation_1720_80be20563f1620981e46.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_2038713cb0876e29c6c8.png media/images/validation_1720_2038713cb0876e29c6c8.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_95d7597f938efa3e944e.png media/images/validation_1720_95d7597f938efa3e944e.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1720_551525fb129586cd0c60.png media/images/validation_1720_551525fb129586cd0c60.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_700fef9298b995bd049b.png media/images/validation_1730_700fef9298b995bd049b.png +2023-07-12 12:35:47,014 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_9fb3356673dd42522db4.png media/images/validation_1730_9fb3356673dd42522db4.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_a001a16142d0cdf3dd8e.png media/images/validation_1730_a001a16142d0cdf3dd8e.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1730_62bcdb68c12bb4eee832.png media/images/validation_1730_62bcdb68c12bb4eee832.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_4dc54de8f062b7449876.png media/images/validation_1740_4dc54de8f062b7449876.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_9ead6b955150561d6d4c.png media/images/validation_1740_9ead6b955150561d6d4c.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_d91846de1cf74e316293.png media/images/validation_1740_d91846de1cf74e316293.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1740_bdda721ad2fd6569a964.png media/images/validation_1740_bdda721ad2fd6569a964.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_1d177e30435b1c654b3d.png media/images/validation_1750_1d177e30435b1c654b3d.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png media/images/validation_1750_5e66d3fcf10d4a6c0cb4.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_06dd5b28c3be7a9bbdb9.png media/images/validation_1750_06dd5b28c3be7a9bbdb9.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1750_94c6bfdf23b465d2a61f.png media/images/validation_1750_94c6bfdf23b465d2a61f.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_88b7298825f502e5a062.png media/images/validation_1760_88b7298825f502e5a062.png +2023-07-12 12:35:47,015 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_0d6a954f28a281ccb3e6.png media/images/validation_1760_0d6a954f28a281ccb3e6.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png media/images/validation_1760_dbc0c05bbbd3b6fb62f7.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1760_e4640ecdc8c22558e54b.png media/images/validation_1760_e4640ecdc8c22558e54b.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_c9724817961bea50e2eb.png media/images/validation_1770_c9724817961bea50e2eb.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f3ed24daee483f81fe21.png media/images/validation_1770_f3ed24daee483f81fe21.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_6f1689c0e9f7dc8526d4.png media/images/validation_1770_6f1689c0e9f7dc8526d4.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1770_f03b061475b754e24cb5.png media/images/validation_1770_f03b061475b754e24cb5.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_e0be2e3dcde4725c11f7.png media/images/validation_1780_e0be2e3dcde4725c11f7.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_43277f4f939e843ab8c9.png media/images/validation_1780_43277f4f939e843ab8c9.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_437f85ccbcb8672d8508.png media/images/validation_1780_437f85ccbcb8672d8508.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1780_19d50aae0838de593b50.png media/images/validation_1780_19d50aae0838de593b50.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_1fbb3840c9621213d4a8.png media/images/validation_1790_1fbb3840c9621213d4a8.png +2023-07-12 12:35:47,016 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_6c8d3f0f2e6689c4c999.png media/images/validation_1790_6c8d3f0f2e6689c4c999.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_73398702cb61fca4ba23.png media/images/validation_1790_73398702cb61fca4ba23.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1790_86559286380435a179bf.png media/images/validation_1790_86559286380435a179bf.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_1b5a6f042a0a4ba4a34a.png media/images/validation_1800_1b5a6f042a0a4ba4a34a.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_3003f5b283d5c2f3d3a3.png media/images/validation_1800_3003f5b283d5c2f3d3a3.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_a5df1ab61da66e19d024.png media/images/validation_1800_a5df1ab61da66e19d024.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1800_c6b5b46f40b7e3cc06e5.png media/images/validation_1800_c6b5b46f40b7e3cc06e5.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_1d8ecdbc961ca85f19f9.png media/images/validation_1810_1d8ecdbc961ca85f19f9.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_22b0716ab99e4f2c1baf.png media/images/validation_1810_22b0716ab99e4f2c1baf.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_db05126b5d482e1f86c1.png media/images/validation_1810_db05126b5d482e1f86c1.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1810_e9390a064f93495e881d.png media/images/validation_1810_e9390a064f93495e881d.png +2023-07-12 12:35:47,017 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_0da500ef601e86506941.png media/images/validation_1820_0da500ef601e86506941.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_7ecf7da22bc3b3fad59d.png media/images/validation_1820_7ecf7da22bc3b3fad59d.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_3619de4ba8713b9bd91c.png media/images/validation_1820_3619de4ba8713b9bd91c.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1820_953f6fe5b812a7050a46.png media/images/validation_1820_953f6fe5b812a7050a46.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b892c3791e1a5536a80.png media/images/validation_1830_7b892c3791e1a5536a80.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_3cff7354bef942c8de30.png media/images/validation_1830_3cff7354bef942c8de30.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_7b9c6d907af1a36abe9c.png media/images/validation_1830_7b9c6d907af1a36abe9c.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1830_d2e0fe71542ddbda5921.png media/images/validation_1830_d2e0fe71542ddbda5921.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_187a1a1f0c43344a0712.png media/images/validation_1840_187a1a1f0c43344a0712.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_31e70925ba342cb958b8.png media/images/validation_1840_31e70925ba342cb958b8.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_2f7e6e14419630be9783.png media/images/validation_1840_2f7e6e14419630be9783.png +2023-07-12 12:35:47,018 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1840_b75b67c6b0d3f3cd134b.png media/images/validation_1840_b75b67c6b0d3f3cd134b.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_82f6b3c68a48fade34f5.png media/images/validation_1850_82f6b3c68a48fade34f5.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_3a3b446f556528dde605.png media/images/validation_1850_3a3b446f556528dde605.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_ec640dd671161721e4f7.png media/images/validation_1850_ec640dd671161721e4f7.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1850_c776faecdec7dde7f595.png media/images/validation_1850_c776faecdec7dde7f595.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_7b660814a00507637354.png media/images/validation_1860_7b660814a00507637354.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2bee3f76993cc3a6ba8f.png media/images/validation_1860_2bee3f76993cc3a6ba8f.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_2ec1726523e7df6ef87a.png media/images/validation_1860_2ec1726523e7df6ef87a.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1860_763a0499812940f346e6.png media/images/validation_1860_763a0499812940f346e6.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_c031f9ce29973bad20f5.png media/images/validation_1870_c031f9ce29973bad20f5.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_386744990d75b0d3adfe.png media/images/validation_1870_386744990d75b0d3adfe.png +2023-07-12 12:35:47,019 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_96924703d0889517e752.png media/images/validation_1870_96924703d0889517e752.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1870_b0e70d9482eb144ce2cd.png media/images/validation_1870_b0e70d9482eb144ce2cd.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_cf1d81e3b206f0eeddf8.png media/images/validation_1880_cf1d81e3b206f0eeddf8.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_e42387963e6467795ac3.png media/images/validation_1880_e42387963e6467795ac3.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_a24606b4de934e6b61c1.png media/images/validation_1880_a24606b4de934e6b61c1.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1880_19c60f744dd634323456.png media/images/validation_1880_19c60f744dd634323456.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_5fabc2eb5b38fc6c441f.png media/images/validation_1890_5fabc2eb5b38fc6c441f.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_362eab872b8e6854bfcb.png media/images/validation_1890_362eab872b8e6854bfcb.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_abe953372839c2e35d11.png media/images/validation_1890_abe953372839c2e35d11.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1890_61843d74c0b1b4039c5a.png media/images/validation_1890_61843d74c0b1b4039c5a.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_755c75927be8fb0bccfb.png media/images/validation_1900_755c75927be8fb0bccfb.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_7ac136e8ccfb72e61b40.png media/images/validation_1900_7ac136e8ccfb72e61b40.png +2023-07-12 12:35:47,020 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_e8722f8bf9b351a33b0d.png media/images/validation_1900_e8722f8bf9b351a33b0d.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1900_1895be2b314f5c3cc922.png media/images/validation_1900_1895be2b314f5c3cc922.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_6fd922b416ffd009954a.png media/images/validation_1910_6fd922b416ffd009954a.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_5760a804d5e3912bf871.png media/images/validation_1910_5760a804d5e3912bf871.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_34349e41d8bc032e1c7d.png media/images/validation_1910_34349e41d8bc032e1c7d.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1910_f7431e028249d68b3d9b.png media/images/validation_1910_f7431e028249d68b3d9b.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_0c56e18802b80a5e1cdb.png media/images/validation_1920_0c56e18802b80a5e1cdb.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_7d921cd7daa3b8f8aae7.png media/images/validation_1920_7d921cd7daa3b8f8aae7.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_60d58608f7bfc9ddde76.png media/images/validation_1920_60d58608f7bfc9ddde76.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1920_3e7c1f6543b9b9815c1e.png media/images/validation_1920_3e7c1f6543b9b9815c1e.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_48056833d0b2d7f2157c.png media/images/validation_1930_48056833d0b2d7f2157c.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_dc6aa83bfa1de0638963.png media/images/validation_1930_dc6aa83bfa1de0638963.png +2023-07-12 12:35:47,021 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_8bf913d97ecc12b8092b.png media/images/validation_1930_8bf913d97ecc12b8092b.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1930_cbdfa1d5c7889c908498.png media/images/validation_1930_cbdfa1d5c7889c908498.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_55fed6437ec58a4992ec.png media/images/validation_1940_55fed6437ec58a4992ec.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_b7870fe1d164423912ca.png media/images/validation_1940_b7870fe1d164423912ca.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_36d3d88a96d60bb7d69e.png media/images/validation_1940_36d3d88a96d60bb7d69e.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1940_42b89bc7a5b6ad127dd2.png media/images/validation_1940_42b89bc7a5b6ad127dd2.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_2c29e26c44ad93206dd5.png media/images/validation_1950_2c29e26c44ad93206dd5.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1d43d5ef09db63ec9f2.png media/images/validation_1950_a1d43d5ef09db63ec9f2.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_a1baa48379524c1c8b17.png media/images/validation_1950_a1baa48379524c1c8b17.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1950_e23c84b831732e39304c.png media/images/validation_1950_e23c84b831732e39304c.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e3afb0220a3fb4fd12f3.png media/images/validation_1960_e3afb0220a3fb4fd12f3.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_35975d12907982332b52.png media/images/validation_1960_35975d12907982332b52.png +2023-07-12 12:35:47,022 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e84cb7c3e576b3bd6e96.png media/images/validation_1960_e84cb7c3e576b3bd6e96.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1960_e96fba72991bb49f94d6.png media/images/validation_1960_e96fba72991bb49f94d6.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_1c5cd897c6da1cf72c60.png media/images/validation_1970_1c5cd897c6da1cf72c60.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_b693a85e0ba7b021e8b5.png media/images/validation_1970_b693a85e0ba7b021e8b5.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_2418660f78d4aba7393a.png media/images/validation_1970_2418660f78d4aba7393a.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1970_f777a51b498e8fce06d9.png media/images/validation_1970_f777a51b498e8fce06d9.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_1db1fd51970584b5daa1.png media/images/validation_1980_1db1fd51970584b5daa1.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_7a8ef7854cf8e37f9cf7.png media/images/validation_1980_7a8ef7854cf8e37f9cf7.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_8caa5ac810c363d03b6e.png media/images/validation_1980_8caa5ac810c363d03b6e.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1980_647b73db1b8d573d85d7.png media/images/validation_1980_647b73db1b8d573d85d7.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_e53b48fcc568a502f437.png media/images/validation_1990_e53b48fcc568a502f437.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_7e8731b3e3747b303dfe.png media/images/validation_1990_7e8731b3e3747b303dfe.png +2023-07-12 12:35:47,023 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_283a622f33c91b0b5af0.png media/images/validation_1990_283a622f33c91b0b5af0.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_1990_35002cfdf6c8ccfffdcd.png media/images/validation_1990_35002cfdf6c8ccfffdcd.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_1f7c7153c03240078b4d.png media/images/validation_2000_1f7c7153c03240078b4d.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_56c478641cd971b63c78.png media/images/validation_2000_56c478641cd971b63c78.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_0669630fb67332c6ded5.png media/images/validation_2000_0669630fb67332c6ded5.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2000_81cc3487bd6c41e95ef1.png media/images/validation_2000_81cc3487bd6c41e95ef1.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_ebb2fde791af714a87fa.png media/images/validation_2010_ebb2fde791af714a87fa.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_fe06a780b0eeb017b688.png media/images/validation_2010_fe06a780b0eeb017b688.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3246141969b6e42324be.png media/images/validation_2010_3246141969b6e42324be.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2010_3fcb388422ea2886399f.png media/images/validation_2010_3fcb388422ea2886399f.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_13af1ccf733709899b51.png media/images/validation_2020_13af1ccf733709899b51.png +2023-07-12 12:35:47,024 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_6f7e3605df1d5a3bcd5d.png media/images/validation_2020_6f7e3605df1d5a3bcd5d.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_ae08a04cbc13fea1f1e4.png media/images/validation_2020_ae08a04cbc13fea1f1e4.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2020_05878893911aa97a5d5b.png media/images/validation_2020_05878893911aa97a5d5b.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_1821049762b6b5d37c08.png media/images/validation_2030_1821049762b6b5d37c08.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_e34ade0521fc9adbaae3.png media/images/validation_2030_e34ade0521fc9adbaae3.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_d0209e06d5c81f80ec0d.png media/images/validation_2030_d0209e06d5c81f80ec0d.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2030_835fcd4e7ec0584630ce.png media/images/validation_2030_835fcd4e7ec0584630ce.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6a4be91a47a6698809a9.png media/images/validation_2040_6a4be91a47a6698809a9.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_167355b47a08bf8109e9.png media/images/validation_2040_167355b47a08bf8109e9.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_7914a4c603a50f2ea240.png media/images/validation_2040_7914a4c603a50f2ea240.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2040_6b091a76f28e18553ef9.png media/images/validation_2040_6b091a76f28e18553ef9.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_95ba8564905e15131c15.png media/images/validation_2050_95ba8564905e15131c15.png +2023-07-12 12:35:47,025 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_025a92e01716e1b46d6d.png media/images/validation_2050_025a92e01716e1b46d6d.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_d4fb051468326f3eb57a.png media/images/validation_2050_d4fb051468326f3eb57a.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2050_8ea42d44ae0516e0ca5e.png media/images/validation_2050_8ea42d44ae0516e0ca5e.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_e0fa444f603a66029fbe.png media/images/validation_2060_e0fa444f603a66029fbe.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_d6f502d4c0d1394b0881.png media/images/validation_2060_d6f502d4c0d1394b0881.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_ad32613282f23549a8bd.png media/images/validation_2060_ad32613282f23549a8bd.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2060_5f6fadce63310a783572.png media/images/validation_2060_5f6fadce63310a783572.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_19ed657f9c10dbd875ca.png media/images/validation_2070_19ed657f9c10dbd875ca.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_e5f36a2a698bd96464b3.png media/images/validation_2070_e5f36a2a698bd96464b3.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_cf0e421ac16cf53c2d5a.png media/images/validation_2070_cf0e421ac16cf53c2d5a.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2070_df0b7fe6b5a5f888eeac.png media/images/validation_2070_df0b7fe6b5a5f888eeac.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_df994cef7f6357e3664e.png media/images/validation_2080_df994cef7f6357e3664e.png +2023-07-12 12:35:47,026 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_4c3033cd3536b3f44094.png media/images/validation_2080_4c3033cd3536b3f44094.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_0d50a284079de0683ad8.png media/images/validation_2080_0d50a284079de0683ad8.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2080_7ed8ebb926937feea5e8.png media/images/validation_2080_7ed8ebb926937feea5e8.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_acf992ec2207b3ea2d60.png media/images/validation_2090_acf992ec2207b3ea2d60.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_f680c2bff58b85913aa3.png media/images/validation_2090_f680c2bff58b85913aa3.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_b72a6b1b7e4871bb5d8f.png media/images/validation_2090_b72a6b1b7e4871bb5d8f.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2090_c267c69f2a522dc95745.png media/images/validation_2090_c267c69f2a522dc95745.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_ed6b49b703996f58696a.png media/images/validation_2100_ed6b49b703996f58696a.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_333e872b5de120d26892.png media/images/validation_2100_333e872b5de120d26892.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_38040cf3cd9cbe1bc314.png media/images/validation_2100_38040cf3cd9cbe1bc314.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2100_2c125071cfc5a2fe04e6.png media/images/validation_2100_2c125071cfc5a2fe04e6.png +2023-07-12 12:35:47,027 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_62fae4a01ba8422d6ed9.png media/images/validation_2110_62fae4a01ba8422d6ed9.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_008df6c26e19a483defd.png media/images/validation_2110_008df6c26e19a483defd.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_3ef95d0764b88f6fbafa.png media/images/validation_2110_3ef95d0764b88f6fbafa.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2110_6a7c69b67225aaafd76d.png media/images/validation_2110_6a7c69b67225aaafd76d.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_b4413d53814dba669bcf.png media/images/validation_2120_b4413d53814dba669bcf.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_7999661823c5d1a194ca.png media/images/validation_2120_7999661823c5d1a194ca.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_1d395d84a1aa765d6c1b.png media/images/validation_2120_1d395d84a1aa765d6c1b.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2120_fe5c8ad23d0df9553dbf.png media/images/validation_2120_fe5c8ad23d0df9553dbf.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_1b4ddf79dc604699f630.png media/images/validation_2130_1b4ddf79dc604699f630.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_259ec0830802f8053fe5.png media/images/validation_2130_259ec0830802f8053fe5.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_bb1524320261cb0216db.png media/images/validation_2130_bb1524320261cb0216db.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2130_0099dae297478bd3ce67.png media/images/validation_2130_0099dae297478bd3ce67.png +2023-07-12 12:35:47,028 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_825c97191445f88d10ad.png media/images/validation_2140_825c97191445f88d10ad.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_ebe038f09b0a2e5c2779.png media/images/validation_2140_ebe038f09b0a2e5c2779.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_69c0860f204684fe6617.png media/images/validation_2140_69c0860f204684fe6617.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2140_a9fe0e5593aba42cc7e6.png media/images/validation_2140_a9fe0e5593aba42cc7e6.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_46dad1ead66f697f0698.png media/images/validation_2150_46dad1ead66f697f0698.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f451e47ddafe0bf1fcc6.png media/images/validation_2150_f451e47ddafe0bf1fcc6.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_f44d5fe284ffc3b74829.png media/images/validation_2150_f44d5fe284ffc3b74829.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2150_4f03c9db92527c876ba9.png media/images/validation_2150_4f03c9db92527c876ba9.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_f78f21bafff0abaf0089.png media/images/validation_2160_f78f21bafff0abaf0089.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_1ee652c4e539d1e07cce.png media/images/validation_2160_1ee652c4e539d1e07cce.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_b97ae163d55862639f70.png media/images/validation_2160_b97ae163d55862639f70.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2160_ec40a3df133e538d57f3.png media/images/validation_2160_ec40a3df133e538d57f3.png +2023-07-12 12:35:47,029 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_36b4d3a784d14fee51a0.png media/images/validation_2170_36b4d3a784d14fee51a0.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_33d957838c074e90f6a2.png media/images/validation_2170_33d957838c074e90f6a2.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_533582fdbffc2ddc97ac.png media/images/validation_2170_533582fdbffc2ddc97ac.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2170_def8ab43612eed95556d.png media/images/validation_2170_def8ab43612eed95556d.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_d63b39c78c18b127efec.png media/images/validation_2180_d63b39c78c18b127efec.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f1a0da8e451df830c88e.png media/images/validation_2180_f1a0da8e451df830c88e.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_5d7322114061dc65484d.png media/images/validation_2180_5d7322114061dc65484d.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2180_f5a9c542fcce2ab79bf5.png media/images/validation_2180_f5a9c542fcce2ab79bf5.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_b89f7b812f07101e7bc5.png media/images/validation_2190_b89f7b812f07101e7bc5.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_51f75fba279dc7d2d626.png media/images/validation_2190_51f75fba279dc7d2d626.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd64ca37330df964f268.png media/images/validation_2190_cd64ca37330df964f268.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2190_cd935f0213029522bb6c.png media/images/validation_2190_cd935f0213029522bb6c.png +2023-07-12 12:35:47,030 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_963ea35b0f8a3f28bf7f.png media/images/validation_2200_963ea35b0f8a3f28bf7f.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_5479be16d55799087c1d.png media/images/validation_2200_5479be16d55799087c1d.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_485475519d4a6c767788.png media/images/validation_2200_485475519d4a6c767788.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2200_8e5e62fe39fe252d734c.png media/images/validation_2200_8e5e62fe39fe252d734c.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_3d504d65d8985e43c4df.png media/images/validation_2210_3d504d65d8985e43c4df.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_7e6627623494cc0c7f15.png media/images/validation_2210_7e6627623494cc0c7f15.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_14e783bea121b92109bd.png media/images/validation_2210_14e783bea121b92109bd.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2210_b4ca94832829e41b0eaa.png media/images/validation_2210_b4ca94832829e41b0eaa.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_fd2b6676675f447dd0bb.png media/images/validation_2220_fd2b6676675f447dd0bb.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_a2b2236edc7a209199e4.png media/images/validation_2220_a2b2236edc7a209199e4.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_be095f7407792a1d8c2b.png media/images/validation_2220_be095f7407792a1d8c2b.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2220_7995e8c15e47416a5a98.png media/images/validation_2220_7995e8c15e47416a5a98.png +2023-07-12 12:35:47,031 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_fa6aae454ea563a1568c.png media/images/validation_2230_fa6aae454ea563a1568c.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_bba8aa2011cbb2b9e998.png media/images/validation_2230_bba8aa2011cbb2b9e998.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_cc1f25e32b609b2e7f13.png media/images/validation_2230_cc1f25e32b609b2e7f13.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2230_c694e9556fc95a54794c.png media/images/validation_2230_c694e9556fc95a54794c.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_c76d157bbb397161c214.png media/images/validation_2240_c76d157bbb397161c214.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_cbb566c39078908ba8b3.png media/images/validation_2240_cbb566c39078908ba8b3.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_17e81b81f9d5ecefe6ad.png media/images/validation_2240_17e81b81f9d5ecefe6ad.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2240_77e784d7b8aacf9759c6.png media/images/validation_2240_77e784d7b8aacf9759c6.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ddcd1a1e8b54f46f7b26.png media/images/validation_2250_ddcd1a1e8b54f46f7b26.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_c1f37bde1c536851ff31.png media/images/validation_2250_c1f37bde1c536851ff31.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png media/images/validation_2250_ecb4d6cde2b6ceae3a0f.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2250_84e331f8cb88258072e5.png media/images/validation_2250_84e331f8cb88258072e5.png +2023-07-12 12:35:47,032 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_1d09dab1e10018c0f5a2.png media/images/validation_2260_1d09dab1e10018c0f5a2.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f5085a2630d7643ccb37.png media/images/validation_2260_f5085a2630d7643ccb37.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_ea013c7b85e22fd83c96.png media/images/validation_2260_ea013c7b85e22fd83c96.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2260_f2fc182f190a6e4d3266.png media/images/validation_2260_f2fc182f190a6e4d3266.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_57fbdf8438f8224664e4.png media/images/validation_2270_57fbdf8438f8224664e4.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_66b7fc2a071b6e92e172.png media/images/validation_2270_66b7fc2a071b6e92e172.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_1db6dc00cd88a44aa4f5.png media/images/validation_2270_1db6dc00cd88a44aa4f5.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2270_506c3fdf835cc8b9e9bc.png media/images/validation_2270_506c3fdf835cc8b9e9bc.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_d6d98a9fc2b412d909b2.png media/images/validation_2280_d6d98a9fc2b412d909b2.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_9e0da1558268d346b2ff.png media/images/validation_2280_9e0da1558268d346b2ff.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_a95379a0e1bb4ee5eefd.png media/images/validation_2280_a95379a0e1bb4ee5eefd.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2280_50d6f6aebffcec48f478.png media/images/validation_2280_50d6f6aebffcec48f478.png +2023-07-12 12:35:47,033 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_dc4de63f9960a0a31886.png media/images/validation_2290_dc4de63f9960a0a31886.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_d8ebb18636c7304a48a5.png media/images/validation_2290_d8ebb18636c7304a48a5.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_8ab18a5e3a00fe86d6ef.png media/images/validation_2290_8ab18a5e3a00fe86d6ef.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2290_f6c9e9af5b280aff421b.png media/images/validation_2290_f6c9e9af5b280aff421b.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_229851204683a32b894d.png media/images/validation_2300_229851204683a32b894d.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_b9647059d362b9b7c491.png media/images/validation_2300_b9647059d362b9b7c491.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png media/images/validation_2300_2b5bd99a5a8e4a5cc6c2.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2300_abce74fb7ce87882e4ef.png media/images/validation_2300_abce74fb7ce87882e4ef.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0b62299b4392d7dab3b5.png media/images/validation_2310_0b62299b4392d7dab3b5.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_8adc40877878e2c5992a.png media/images/validation_2310_8adc40877878e2c5992a.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_b8447334c2fa99557dab.png media/images/validation_2310_b8447334c2fa99557dab.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2310_0a4470d06e9e5634fc30.png media/images/validation_2310_0a4470d06e9e5634fc30.png +2023-07-12 12:35:47,034 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_6e0bdeefa42f0abef07e.png media/images/validation_2320_6e0bdeefa42f0abef07e.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_cad2712e0579795eae61.png media/images/validation_2320_cad2712e0579795eae61.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_8982b4ca24e18e3a2e66.png media/images/validation_2320_8982b4ca24e18e3a2e66.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2320_d536893289030f727428.png media/images/validation_2320_d536893289030f727428.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_9d435e637b3005c16f38.png media/images/validation_2330_9d435e637b3005c16f38.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_c9b9321adf648747a0cb.png media/images/validation_2330_c9b9321adf648747a0cb.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_86f3d365c9fc4e6ab5e8.png media/images/validation_2330_86f3d365c9fc4e6ab5e8.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2330_1afc2708fa415a3e0f40.png media/images/validation_2330_1afc2708fa415a3e0f40.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_e1920e85b34fdfb6609a.png media/images/validation_2340_e1920e85b34fdfb6609a.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_65d2e2217439693f9468.png media/images/validation_2340_65d2e2217439693f9468.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_cfd6e181a1bb997a54d6.png media/images/validation_2340_cfd6e181a1bb997a54d6.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2340_d7a930a7a71d94c1211f.png media/images/validation_2340_d7a930a7a71d94c1211f.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_576ed58f083c4925ff4f.png media/images/validation_2350_576ed58f083c4925ff4f.png +2023-07-12 12:35:47,035 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_6b91f2effcbccf59aafb.png media/images/validation_2350_6b91f2effcbccf59aafb.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_7116b69ba382286251a6.png media/images/validation_2350_7116b69ba382286251a6.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2350_4776174cbccf4dcad7cf.png media/images/validation_2350_4776174cbccf4dcad7cf.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_3d170b31330953e821ba.png media/images/validation_2360_3d170b31330953e821ba.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_6f1a9a30ce0054e1f213.png media/images/validation_2360_6f1a9a30ce0054e1f213.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_a603ae3e847019132432.png media/images/validation_2360_a603ae3e847019132432.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2360_4eb285dbb680a44d3c77.png media/images/validation_2360_4eb285dbb680a44d3c77.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_6b60edf29451c64b6e2b.png media/images/validation_2370_6b60edf29451c64b6e2b.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_e3cc962dadab1208b51d.png media/images/validation_2370_e3cc962dadab1208b51d.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_4fd19c0f067294c7a9e6.png media/images/validation_2370_4fd19c0f067294c7a9e6.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2370_81faade70bf66d35a629.png media/images/validation_2370_81faade70bf66d35a629.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_841e12bb766b52954d05.png media/images/validation_2380_841e12bb766b52954d05.png +2023-07-12 12:35:47,036 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_3b3663922cc96b138671.png media/images/validation_2380_3b3663922cc96b138671.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_f14220ee5227d3539481.png media/images/validation_2380_f14220ee5227d3539481.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2380_b3144dac275fe828dece.png media/images/validation_2380_b3144dac275fe828dece.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_34c10cf3cd6d89963a05.png media/images/validation_2390_34c10cf3cd6d89963a05.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_92717b0771dd961acad8.png media/images/validation_2390_92717b0771dd961acad8.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_674a3618e81973b0470a.png media/images/validation_2390_674a3618e81973b0470a.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2390_0c825ba6ed95b9246d26.png media/images/validation_2390_0c825ba6ed95b9246d26.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_7fcaf32b847afe6bc2ca.png media/images/validation_2400_7fcaf32b847afe6bc2ca.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_3cef30f8c2574b5c86c0.png media/images/validation_2400_3cef30f8c2574b5c86c0.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png media/images/validation_2400_80b2f8ce3fa60fa9b5b1.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2400_a254fe937a4ee82dda50.png media/images/validation_2400_a254fe937a4ee82dda50.png +2023-07-12 12:35:47,037 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_f055f10be828e22bd53d.png media/images/validation_2410_f055f10be828e22bd53d.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_fcac0499a5c4fff7482f.png media/images/validation_2410_fcac0499a5c4fff7482f.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5eb323fa89395b0ed41c.png media/images/validation_2410_5eb323fa89395b0ed41c.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2410_5870baca82c8f1ea9c65.png media/images/validation_2410_5870baca82c8f1ea9c65.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_e300a087119470bd19f7.png media/images/validation_2420_e300a087119470bd19f7.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_a3ef41aa53f7542e2ff6.png media/images/validation_2420_a3ef41aa53f7542e2ff6.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_86a10e9e981b83c54181.png media/images/validation_2420_86a10e9e981b83c54181.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2420_508d57a7914d9268a798.png media/images/validation_2420_508d57a7914d9268a798.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_d23160faa6354332ac90.png media/images/validation_2430_d23160faa6354332ac90.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_e3cbf88a5d454b013c98.png media/images/validation_2430_e3cbf88a5d454b013c98.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_bdfc2304f95d55e3c519.png media/images/validation_2430_bdfc2304f95d55e3c519.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2430_9d7fc07bee22787a2007.png media/images/validation_2430_9d7fc07bee22787a2007.png +2023-07-12 12:35:47,038 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_f15d4e86eefcf39e1eae.png media/images/validation_2440_f15d4e86eefcf39e1eae.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_8826c1292b54e84dc27b.png media/images/validation_2440_8826c1292b54e84dc27b.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_02cad4f86332c4d30e0f.png media/images/validation_2440_02cad4f86332c4d30e0f.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2440_b90423e76bb46ed0bd51.png media/images/validation_2440_b90423e76bb46ed0bd51.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_f4dd09917aefe94149cf.png media/images/validation_2450_f4dd09917aefe94149cf.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_807425807eddcc0e0aaa.png media/images/validation_2450_807425807eddcc0e0aaa.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_30096936a6475e462a39.png media/images/validation_2450_30096936a6475e462a39.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2450_8fe94ed8c310dae4cd3c.png media/images/validation_2450_8fe94ed8c310dae4cd3c.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_d5445284771d7b2c66a0.png media/images/validation_2460_d5445284771d7b2c66a0.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_c867e6da32e13d963e59.png media/images/validation_2460_c867e6da32e13d963e59.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_815ded56ecd664726052.png media/images/validation_2460_815ded56ecd664726052.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2460_e5fd4b41b8c0c33c93ac.png media/images/validation_2460_e5fd4b41b8c0c33c93ac.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_edd32e44eaf81373d115.png media/images/validation_2470_edd32e44eaf81373d115.png +2023-07-12 12:35:47,039 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_db2b2e82dd30d35d1311.png media/images/validation_2470_db2b2e82dd30d35d1311.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_8dcd67f2db2d9902e40b.png media/images/validation_2470_8dcd67f2db2d9902e40b.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2470_97ab1f90f62f7fd81135.png media/images/validation_2470_97ab1f90f62f7fd81135.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_b53f3dded240b500b1cb.png media/images/validation_2480_b53f3dded240b500b1cb.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_a061712ad948dbaccbd5.png media/images/validation_2480_a061712ad948dbaccbd5.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_129f224b0015451a7862.png media/images/validation_2480_129f224b0015451a7862.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2480_6c96769927607cfa02a7.png media/images/validation_2480_6c96769927607cfa02a7.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_e2238e57e86ad9eb1236.png media/images/validation_2490_e2238e57e86ad9eb1236.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fbe171c1140c25cd1cb2.png media/images/validation_2490_fbe171c1140c25cd1cb2.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_eb52c1f3852b2db713d6.png media/images/validation_2490_eb52c1f3852b2db713d6.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2490_fb4911582a74ff6e59f1.png media/images/validation_2490_fb4911582a74ff6e59f1.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_1a8029b03d2f4e8ba6b5.png media/images/validation_2500_1a8029b03d2f4e8ba6b5.png +2023-07-12 12:35:47,040 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_a3765156a53105d7a44f.png media/images/validation_2500_a3765156a53105d7a44f.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_7b6d1552c5731ce4d99a.png media/images/validation_2500_7b6d1552c5731ce4d99a.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2500_45f174c3353c67e7d2fa.png media/images/validation_2500_45f174c3353c67e7d2fa.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_f73a3d21c0ad929cc0df.png media/images/validation_2510_f73a3d21c0ad929cc0df.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png media/images/validation_2510_ce5fb7efdd3e17eaf8a3.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_2345ba38c318660ea20d.png media/images/validation_2510_2345ba38c318660ea20d.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2510_4acac46bf9cdc8077cd8.png media/images/validation_2510_4acac46bf9cdc8077cd8.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_ce5f5d922b12cb9a8e58.png media/images/validation_2520_ce5f5d922b12cb9a8e58.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_55683596f533718ecbc7.png media/images/validation_2520_55683596f533718ecbc7.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_9cd53a6492dff1119fcd.png media/images/validation_2520_9cd53a6492dff1119fcd.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2520_4e5b7eac0474303b7db6.png media/images/validation_2520_4e5b7eac0474303b7db6.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_a5bdcb2b01b23f824fcf.png media/images/validation_2530_a5bdcb2b01b23f824fcf.png +2023-07-12 12:35:47,041 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_e419bc8806d8bdc57fa0.png media/images/validation_2530_e419bc8806d8bdc57fa0.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_48e39b87d59cc06724b3.png media/images/validation_2530_48e39b87d59cc06724b3.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2530_4436a3e5cc352d2a1017.png media/images/validation_2530_4436a3e5cc352d2a1017.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_03d4142985e8dc5f61fc.png media/images/validation_2540_03d4142985e8dc5f61fc.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_13c9cc17dc94ed86351c.png media/images/validation_2540_13c9cc17dc94ed86351c.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_97a6019e1743b1256999.png media/images/validation_2540_97a6019e1743b1256999.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2540_1248be5f76e40ac7db9c.png media/images/validation_2540_1248be5f76e40ac7db9c.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_8cdd64e4abd06459cc31.png media/images/validation_2550_8cdd64e4abd06459cc31.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_5f9749e51f806c3a7042.png media/images/validation_2550_5f9749e51f806c3a7042.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_b1d38d2764a5721d15ac.png media/images/validation_2550_b1d38d2764a5721d15ac.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2550_7e913201741ac8370fe9.png media/images/validation_2550_7e913201741ac8370fe9.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_35fe8e67bdbd375bad51.png media/images/validation_2560_35fe8e67bdbd375bad51.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_e4060fd5427ae0ecee26.png media/images/validation_2560_e4060fd5427ae0ecee26.png +2023-07-12 12:35:47,042 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8ef00575ef619617eb8f.png media/images/validation_2560_8ef00575ef619617eb8f.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2560_8c7622cbbe1b8f2266c2.png media/images/validation_2560_8c7622cbbe1b8f2266c2.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_5dcd5f49c5610ed98e57.png media/images/validation_2570_5dcd5f49c5610ed98e57.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_ceacc8064824cf24b9c3.png media/images/validation_2570_ceacc8064824cf24b9c3.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_13fad504fb5c2ac0ee04.png media/images/validation_2570_13fad504fb5c2ac0ee04.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2570_0287eb8f3f80591bc234.png media/images/validation_2570_0287eb8f3f80591bc234.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_784615f77a2b4e7047a9.png media/images/validation_2580_784615f77a2b4e7047a9.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_38ab10654e678fd3808d.png media/images/validation_2580_38ab10654e678fd3808d.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_0362f6603354bb565423.png media/images/validation_2580_0362f6603354bb565423.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2580_12a798eb5ac3d54ffad6.png media/images/validation_2580_12a798eb5ac3d54ffad6.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_62ad7e090c150c4642cd.png media/images/validation_2590_62ad7e090c150c4642cd.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_2342de4c23f89a71e70a.png media/images/validation_2590_2342de4c23f89a71e70a.png +2023-07-12 12:35:47,043 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_12a6034cf8cfc23b5df1.png media/images/validation_2590_12a6034cf8cfc23b5df1.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2590_59ef69c8ce6d4ab2107c.png media/images/validation_2590_59ef69c8ce6d4ab2107c.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_404750014d2070b4b00e.png media/images/validation_2600_404750014d2070b4b00e.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_e5c153d3981bac3820e6.png media/images/validation_2600_e5c153d3981bac3820e6.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_305e21c628b26bb7d55e.png media/images/validation_2600_305e21c628b26bb7d55e.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2600_5996b8fa55b566dccbed.png media/images/validation_2600_5996b8fa55b566dccbed.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_a3c7e4b8847949aa758f.png media/images/validation_2610_a3c7e4b8847949aa758f.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_d8ef17b50203820bbbe3.png media/images/validation_2610_d8ef17b50203820bbbe3.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_5b8b7b248eeeadd01dc0.png media/images/validation_2610_5b8b7b248eeeadd01dc0.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2610_3c27c596a729886e6078.png media/images/validation_2610_3c27c596a729886e6078.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_8d036206f070328af8f6.png media/images/validation_2620_8d036206f070328af8f6.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_b1cd6dc23fe6b08880be.png media/images/validation_2620_b1cd6dc23fe6b08880be.png +2023-07-12 12:35:47,044 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_792f17b1a307463180d2.png media/images/validation_2620_792f17b1a307463180d2.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2620_87526415e87acfa56b82.png media/images/validation_2620_87526415e87acfa56b82.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_af826553a0f3457c21a4.png media/images/validation_2630_af826553a0f3457c21a4.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_1e363e4fe52a8dd1f3d5.png media/images/validation_2630_1e363e4fe52a8dd1f3d5.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_01eb7980170d3a100648.png media/images/validation_2630_01eb7980170d3a100648.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2630_f752872a939e31b6a407.png media/images/validation_2630_f752872a939e31b6a407.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_cbe3a2e2923fea324bbe.png media/images/validation_2640_cbe3a2e2923fea324bbe.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_ece7f7211bafcdaa62b2.png media/images/validation_2640_ece7f7211bafcdaa62b2.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_8852be70b2d4d0b9cda6.png media/images/validation_2640_8852be70b2d4d0b9cda6.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2640_60a5b23b74be955f2c27.png media/images/validation_2640_60a5b23b74be955f2c27.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_e1afc2c90dd83c53afe8.png media/images/validation_2650_e1afc2c90dd83c53afe8.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_3a9f356aa83d5b8c3154.png media/images/validation_2650_3a9f356aa83d5b8c3154.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_feb61af945632b2e5a0d.png media/images/validation_2650_feb61af945632b2e5a0d.png +2023-07-12 12:35:47,045 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2650_bd16c2dda6cb242546bd.png media/images/validation_2650_bd16c2dda6cb242546bd.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_39f7775ce6273f5abdd3.png media/images/validation_2660_39f7775ce6273f5abdd3.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_be22a91ba76075904920.png media/images/validation_2660_be22a91ba76075904920.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_570101fe2f3838813987.png media/images/validation_2660_570101fe2f3838813987.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2660_049933627fe17227a897.png media/images/validation_2660_049933627fe17227a897.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_b7c6a59e577b6aff9423.png media/images/validation_2670_b7c6a59e577b6aff9423.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_8f7a128a9aebab91d68e.png media/images/validation_2670_8f7a128a9aebab91d68e.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_abd6457914db5ecb11f5.png media/images/validation_2670_abd6457914db5ecb11f5.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2670_81b157e520543297899b.png media/images/validation_2670_81b157e520543297899b.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_3ca3ef00974e8561788d.png media/images/validation_2680_3ca3ef00974e8561788d.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_7cb40a3f835111c25bc8.png media/images/validation_2680_7cb40a3f835111c25bc8.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_1128a7fc997078f6857d.png media/images/validation_2680_1128a7fc997078f6857d.png +2023-07-12 12:35:47,046 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2680_5968f013df449c836103.png media/images/validation_2680_5968f013df449c836103.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_f6facb6c2b52824b8087.png media/images/validation_2690_f6facb6c2b52824b8087.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_ea076148ce2508581f4a.png media/images/validation_2690_ea076148ce2508581f4a.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_6b92b33ac5d9538cd14f.png media/images/validation_2690_6b92b33ac5d9538cd14f.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2690_09839606b108aef4ba45.png media/images/validation_2690_09839606b108aef4ba45.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_2a9c6e3e9705c484c43d.png media/images/validation_2700_2a9c6e3e9705c484c43d.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_951a6312bd4ff445e3bf.png media/images/validation_2700_951a6312bd4ff445e3bf.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_cd74dc827e683d49ff49.png media/images/validation_2700_cd74dc827e683d49ff49.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2700_9d44392c5a9c6cbb63a6.png media/images/validation_2700_9d44392c5a9c6cbb63a6.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_de3f2b25e35e4537c17e.png media/images/validation_2710_de3f2b25e35e4537c17e.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_5ca25be3dc62ed26aa0d.png media/images/validation_2710_5ca25be3dc62ed26aa0d.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_1328ec76d8379d97a378.png media/images/validation_2710_1328ec76d8379d97a378.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2710_6e2b6aa87197c3eba430.png media/images/validation_2710_6e2b6aa87197c3eba430.png +2023-07-12 12:35:47,047 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_fb1e12ca3be46bca7950.png media/images/validation_2720_fb1e12ca3be46bca7950.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_cab768f5b1c4da57e828.png media/images/validation_2720_cab768f5b1c4da57e828.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_9ebc2d1c275cb06de97e.png media/images/validation_2720_9ebc2d1c275cb06de97e.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2720_62fa4b4be6b75191315d.png media/images/validation_2720_62fa4b4be6b75191315d.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_468e29d1792ddfc31f96.png media/images/validation_2730_468e29d1792ddfc31f96.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_ec678e94be72a14ad671.png media/images/validation_2730_ec678e94be72a14ad671.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_7a6b051b3cd7a36b4c2e.png media/images/validation_2730_7a6b051b3cd7a36b4c2e.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2730_a1852227cbd29c30acc0.png media/images/validation_2730_a1852227cbd29c30acc0.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_a1a975ef1b62e7916638.png media/images/validation_2740_a1a975ef1b62e7916638.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_3a9de177b9a56588f2fd.png media/images/validation_2740_3a9de177b9a56588f2fd.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_88328b2a91d7dcdc342e.png media/images/validation_2740_88328b2a91d7dcdc342e.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2740_1eadc0d7d441ac97c181.png media/images/validation_2740_1eadc0d7d441ac97c181.png +2023-07-12 12:35:47,048 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_809fb9a00d33011ef1b7.png media/images/validation_2750_809fb9a00d33011ef1b7.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_35d816078fba6fbc5f20.png media/images/validation_2750_35d816078fba6fbc5f20.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_e2650e26371a6e674cfa.png media/images/validation_2750_e2650e26371a6e674cfa.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2750_44058a9b1a42011478fa.png media/images/validation_2750_44058a9b1a42011478fa.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_70b32648987a29c582d5.png media/images/validation_2760_70b32648987a29c582d5.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_aab6a0c7b6bb6712ffaa.png media/images/validation_2760_aab6a0c7b6bb6712ffaa.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png media/images/validation_2760_0c6b6bbd0b4e3c1e9a49.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2760_0eccab278b3c49ff65d0.png media/images/validation_2760_0eccab278b3c49ff65d0.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_a6f733cd68a255bc3547.png media/images/validation_2770_a6f733cd68a255bc3547.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_6c225fe157884e440759.png media/images/validation_2770_6c225fe157884e440759.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b06c0ff3bb6bd189e084.png media/images/validation_2770_b06c0ff3bb6bd189e084.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2770_b32e150d333150b6d60d.png media/images/validation_2770_b32e150d333150b6d60d.png +2023-07-12 12:35:47,049 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37df37da56d5d142423a.png media/images/validation_2780_37df37da56d5d142423a.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_0ebabc283025333f43a1.png media/images/validation_2780_0ebabc283025333f43a1.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_5cdc5d44821692ab8816.png media/images/validation_2780_5cdc5d44821692ab8816.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2780_37cb82e9b9e53c9e5c5d.png media/images/validation_2780_37cb82e9b9e53c9e5c5d.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_5a82d28324cd9b502db5.png media/images/validation_2790_5a82d28324cd9b502db5.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_1df5c10fc9ddab2abd27.png media/images/validation_2790_1df5c10fc9ddab2abd27.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_a3c09a1cdfb2790cb544.png media/images/validation_2790_a3c09a1cdfb2790cb544.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2790_b0ea698af085bca141c6.png media/images/validation_2790_b0ea698af085bca141c6.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_0a4eb1c4607d5a5fd63e.png media/images/validation_2800_0a4eb1c4607d5a5fd63e.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_a4a930c3aca9ed6b1e66.png media/images/validation_2800_a4a930c3aca9ed6b1e66.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_1e810046ab391ec21457.png media/images/validation_2800_1e810046ab391ec21457.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2800_55e6ce392e83cd6c9775.png media/images/validation_2800_55e6ce392e83cd6c9775.png +2023-07-12 12:35:47,050 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_79746b73a2d51ab398aa.png media/images/validation_2810_79746b73a2d51ab398aa.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8fd491043a269feded86.png media/images/validation_2810_8fd491043a269feded86.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_8e4329d82c9013ca58f7.png media/images/validation_2810_8e4329d82c9013ca58f7.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2810_1f22abcfa3a067c7928c.png media/images/validation_2810_1f22abcfa3a067c7928c.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_0393c6fc4db3e318d3d3.png media/images/validation_2820_0393c6fc4db3e318d3d3.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_4fee48092f2ede4f3c25.png media/images/validation_2820_4fee48092f2ede4f3c25.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_ba69b7651ab0dcd73bdc.png media/images/validation_2820_ba69b7651ab0dcd73bdc.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2820_af052cd0179c44de7cb7.png media/images/validation_2820_af052cd0179c44de7cb7.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_a5847a1c9ac576d3c242.png media/images/validation_2830_a5847a1c9ac576d3c242.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_d0edf809dce9067d913a.png media/images/validation_2830_d0edf809dce9067d913a.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_1176c8bf7fff463bf9c6.png media/images/validation_2830_1176c8bf7fff463bf9c6.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2830_f3f42317c0356c47ff20.png media/images/validation_2830_f3f42317c0356c47ff20.png +2023-07-12 12:35:47,051 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_49d3966f62f54fe9ca62.png media/images/validation_2840_49d3966f62f54fe9ca62.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_a708be3afa6ae1a86e60.png media/images/validation_2840_a708be3afa6ae1a86e60.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_7e7433279443a3a2bfa6.png media/images/validation_2840_7e7433279443a3a2bfa6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2840_2d3f82af7844062f4b35.png media/images/validation_2840_2d3f82af7844062f4b35.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_dfe649466fc1947dac85.png media/images/validation_2850_dfe649466fc1947dac85.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_2975002e2263b7b13da6.png media/images/validation_2850_2975002e2263b7b13da6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_462a59d1382cf36adaec.png media/images/validation_2850_462a59d1382cf36adaec.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2850_9c5803c959d6e590aeae.png media/images/validation_2850_9c5803c959d6e590aeae.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_c18b3c294fb48283f9a6.png media/images/validation_2860_c18b3c294fb48283f9a6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_e995c389d61465ef369b.png media/images/validation_2860_e995c389d61465ef369b.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_a6d367b1ce47fd112b6d.png media/images/validation_2860_a6d367b1ce47fd112b6d.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2860_556f808727f18f3fb7b6.png media/images/validation_2860_556f808727f18f3fb7b6.png +2023-07-12 12:35:47,052 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_bcac2792455fc0c610c6.png media/images/validation_2870_bcac2792455fc0c610c6.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_0e8bce2d57e7e047c357.png media/images/validation_2870_0e8bce2d57e7e047c357.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_fe5c449ea1876a5e69ca.png media/images/validation_2870_fe5c449ea1876a5e69ca.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2870_1d174f0fc051b4fa542a.png media/images/validation_2870_1d174f0fc051b4fa542a.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_9b8a7d4902a70bbe3529.png media/images/validation_2880_9b8a7d4902a70bbe3529.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_948197ed3bdc8486cb6a.png media/images/validation_2880_948197ed3bdc8486cb6a.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_7ab6e23c2268ebda9ab0.png media/images/validation_2880_7ab6e23c2268ebda9ab0.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2880_ad77bbc22d61066c6ce9.png media/images/validation_2880_ad77bbc22d61066c6ce9.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_cb1bd764507bb8c18d8a.png media/images/validation_2890_cb1bd764507bb8c18d8a.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_8578482fbd741b1fda1e.png media/images/validation_2890_8578482fbd741b1fda1e.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_98e0fea80621818b19b9.png media/images/validation_2890_98e0fea80621818b19b9.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2890_5371e29c2e2d3a7d3242.png media/images/validation_2890_5371e29c2e2d3a7d3242.png +2023-07-12 12:35:47,053 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_3d0ad7c57f2af82f8bbc.png media/images/validation_2900_3d0ad7c57f2af82f8bbc.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_8f1fe1b72bfe04a0b014.png media/images/validation_2900_8f1fe1b72bfe04a0b014.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_aa4b86f6ab2a980e6525.png media/images/validation_2900_aa4b86f6ab2a980e6525.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2900_27c9042e9f887913b3c0.png media/images/validation_2900_27c9042e9f887913b3c0.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_2d10951a0f8b9aed5734.png media/images/validation_2910_2d10951a0f8b9aed5734.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_42b541158c5e9a2b90ee.png media/images/validation_2910_42b541158c5e9a2b90ee.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_ba216e11217e75e9719b.png media/images/validation_2910_ba216e11217e75e9719b.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2910_885722981f42772a687d.png media/images/validation_2910_885722981f42772a687d.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_3f89f7c3f5ff7126de27.png media/images/validation_2920_3f89f7c3f5ff7126de27.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_ec96bb36cf4e66588cd4.png media/images/validation_2920_ec96bb36cf4e66588cd4.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_e0997462fe5eec3ca9d3.png media/images/validation_2920_e0997462fe5eec3ca9d3.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2920_0a30f7284bacae8356b3.png media/images/validation_2920_0a30f7284bacae8356b3.png +2023-07-12 12:35:47,054 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_fd71982266285f250c3c.png media/images/validation_2930_fd71982266285f250c3c.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_2268465f1770440e1c31.png media/images/validation_2930_2268465f1770440e1c31.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_94f80d2b1ffdc85aabfa.png media/images/validation_2930_94f80d2b1ffdc85aabfa.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2930_ccf35b7fa382262ad056.png media/images/validation_2930_ccf35b7fa382262ad056.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_11bcda97ccccef6067ba.png media/images/validation_2940_11bcda97ccccef6067ba.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_ba511e69b1b4757bb114.png media/images/validation_2940_ba511e69b1b4757bb114.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_32467ef9d42b5e1e1a33.png media/images/validation_2940_32467ef9d42b5e1e1a33.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2940_c49f0237624dac615f66.png media/images/validation_2940_c49f0237624dac615f66.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_5189a1941e0ee2300ecf.png media/images/validation_2950_5189a1941e0ee2300ecf.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_dce90c9fdac2309feb19.png media/images/validation_2950_dce90c9fdac2309feb19.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_0d199df0c17d88c8726a.png media/images/validation_2950_0d199df0c17d88c8726a.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2950_626d6d1dadf7aad42361.png media/images/validation_2950_626d6d1dadf7aad42361.png +2023-07-12 12:35:47,055 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_7b43af642a476832b780.png media/images/validation_2960_7b43af642a476832b780.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_76db5a33fad883eea4bc.png media/images/validation_2960_76db5a33fad883eea4bc.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_366cee911e557f14b4e5.png media/images/validation_2960_366cee911e557f14b4e5.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2960_95e48c47882f1ab9ab69.png media/images/validation_2960_95e48c47882f1ab9ab69.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_c7e2d3867abfa6c74ec7.png media/images/validation_2970_c7e2d3867abfa6c74ec7.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_da274c6af8f1eb4795bd.png media/images/validation_2970_da274c6af8f1eb4795bd.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_3fff3d3522577450efef.png media/images/validation_2970_3fff3d3522577450efef.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2970_949364012c78ad26092f.png media/images/validation_2970_949364012c78ad26092f.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_86a486a36860b5c6245c.png media/images/validation_2980_86a486a36860b5c6245c.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_18ea565e486f4bef2bbd.png media/images/validation_2980_18ea565e486f4bef2bbd.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_dfadefec7519a6555efa.png media/images/validation_2980_dfadefec7519a6555efa.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2980_26cc58965abb0300912c.png media/images/validation_2980_26cc58965abb0300912c.png +2023-07-12 12:35:47,056 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_d1566b7d6fc6fface55f.png media/images/validation_2990_d1566b7d6fc6fface55f.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_3375e10d6215e10e1662.png media/images/validation_2990_3375e10d6215e10e1662.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_60ce9836f585a7205184.png media/images/validation_2990_60ce9836f585a7205184.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_2990_7a4d6b6bbbebe28cdf93.png media/images/validation_2990_7a4d6b6bbbebe28cdf93.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_88582cfed2f573568e58.png media/images/validation_3000_88582cfed2f573568e58.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png media/images/validation_3000_ca0ca2e8eda6eed4a6ce.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_400a50fac321ed4c2979.png media/images/validation_3000_400a50fac321ed4c2979.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [dir_watcher.py:finish():403] scan save: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/media/images/validation_3000_72521f709e19fb5db039.png media/images/validation_3000_72521f709e19fb5db039.png +2023-07-12 12:35:47,057 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 10 +2023-07-12 12:35:47,057 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:47,059 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:47,059 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 10 +2023-07-12 12:35:47,059 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:47,060 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 10 +2023-07-12 12:35:47,060 INFO SenderThread:3148485 [file_pusher.py:finish():159] shutting down file pusher +2023-07-12 12:35:47,340 INFO wandb-upload_0:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/config.yaml +2023-07-12 12:35:47,349 INFO wandb-upload_1:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/requirements.txt +2023-07-12 12:35:47,354 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:47,354 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:47,367 INFO wandb-upload_3:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/wandb-summary.json +2023-07-12 12:35:47,379 INFO wandb-upload_2:3148485 [upload_job.py:push():133] Uploaded file /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/files/output.log +2023-07-12 12:35:47,579 INFO Thread-11 :3148485 [sender.py:transition_state():602] send defer: 11 +2023-07-12 12:35:47,580 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:47,580 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 11 +2023-07-12 12:35:47,581 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:47,581 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 11 +2023-07-12 12:35:47,581 INFO SenderThread:3148485 [file_pusher.py:join():164] waiting for file pusher +2023-07-12 12:35:47,581 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 12 +2023-07-12 12:35:47,581 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:47,581 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 12 +2023-07-12 12:35:47,582 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:47,582 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 12 +2023-07-12 12:35:48,314 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 13 +2023-07-12 12:35:48,314 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:48,314 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 13 +2023-07-12 12:35:48,315 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:48,315 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 13 +2023-07-12 12:35:48,315 INFO SenderThread:3148485 [sender.py:transition_state():602] send defer: 14 +2023-07-12 12:35:48,316 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: defer +2023-07-12 12:35:48,316 INFO HandlerThread:3148485 [handler.py:handle_request_defer():170] handle defer: 14 +2023-07-12 12:35:48,316 DEBUG SenderThread:3148485 [sender.py:send():369] send: final +2023-07-12 12:35:48,317 DEBUG SenderThread:3148485 [sender.py:send():369] send: footer +2023-07-12 12:35:48,317 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: defer +2023-07-12 12:35:48,317 INFO SenderThread:3148485 [sender.py:send_request_defer():598] handle sender defer: 14 +2023-07-12 12:35:48,318 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:48,318 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:48,322 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: poll_exit +2023-07-12 12:35:48,323 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: poll_exit +2023-07-12 12:35:48,325 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: server_info +2023-07-12 12:35:48,325 DEBUG SenderThread:3148485 [sender.py:send_request():396] send_request: server_info +2023-07-12 12:35:48,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: get_summary +2023-07-12 12:35:48,328 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: sampled_history +2023-07-12 12:35:48,481 INFO MainThread:3148485 [wandb_run.py:_footer_history_summary_info():3467] rendering history +2023-07-12 12:35:48,481 INFO MainThread:3148485 [wandb_run.py:_footer_history_summary_info():3499] rendering summary +2023-07-12 12:35:48,482 INFO MainThread:3148485 [wandb_run.py:_footer_sync_info():3426] logging synced files +2023-07-12 12:35:48,482 DEBUG HandlerThread:3148485 [handler.py:handle_request():144] handle_request: shutdown +2023-07-12 12:35:48,482 INFO HandlerThread:3148485 [handler.py:finish():854] shutting down handler +2023-07-12 12:35:49,325 INFO WriterThread:3148485 [datastore.py:close():298] close: /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb +2023-07-12 12:35:49,481 INFO SenderThread:3148485 [sender.py:finish():1526] shutting down sender +2023-07-12 12:35:49,481 INFO SenderThread:3148485 [file_pusher.py:finish():159] shutting down file pusher +2023-07-12 12:35:49,481 INFO SenderThread:3148485 [file_pusher.py:join():164] waiting for file pusher diff --git a/wandb/run-20230712_000308-yuype6tc/logs/debug.log b/wandb/run-20230712_000308-yuype6tc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..02e5db1bef6bfe0a62a90637469ea3bf8bf52bdd --- /dev/null +++ b/wandb/run-20230712_000308-yuype6tc/logs/debug.log @@ -0,0 +1,42 @@ +2023-07-12 00:03:08,079 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4 +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Configure stats pid to 3148323 +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'} +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_init.py:_log_setup():507] Logging user logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/logs/debug.log +2023-07-12 00:03:08,080 INFO MainThread:3148323 [wandb_init.py:_log_setup():508] Logging internal logs to /l/vision/v5/sragas/easel_ai/models_plain/wandb/run-20230712_000308-yuype6tc/logs/debug-internal.log +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():547] calling init triggers +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():554] wandb.init called with sweep_config: {} +config: {} +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():596] starting backend +2023-07-12 00:03:08,081 INFO MainThread:3148323 [wandb_init.py:init():600] setting up manager +2023-07-12 00:03:08,084 INFO MainThread:3148323 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-07-12 00:03:08,087 INFO MainThread:3148323 [wandb_init.py:init():606] backend started and connected +2023-07-12 00:03:08,091 INFO MainThread:3148323 [wandb_init.py:init():703] updated telemetry +2023-07-12 00:03:08,092 INFO MainThread:3148323 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout +2023-07-12 00:03:08,319 INFO MainThread:3148323 [wandb_run.py:_on_init():2176] communicating current version +2023-07-12 00:03:08,412 INFO MainThread:3148323 [wandb_run.py:_on_init():2185] got version response upgrade_message: "wandb version 0.15.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2023-07-12 00:03:08,412 INFO MainThread:3148323 [wandb_init.py:init():787] starting run threads in backend +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_console_start():2155] atexit reg +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_redirect():2075] Wrapping output streams. +2023-07-12 00:03:08,572 INFO MainThread:3148323 [wandb_run.py:_redirect():2100] Redirects installed. +2023-07-12 00:03:08,573 INFO MainThread:3148323 [wandb_init.py:init():828] run started, returning control to user process +2023-07-12 00:03:18,853 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4 +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Configure stats pid to 3148323 +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /u/sragas/.config/wandb/settings +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from /nfs/nfs2/home/sragas/demo/wandb/settings +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train_text_to_image_lora.py', 'program': 'train_text_to_image_lora.py'} +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_init.py:_log_setup():507] Logging user logs to /nfs/nfs2/home/sragas/demo/wandb/run-20230712_000318-e287bzfl/logs/debug.log +2023-07-12 00:03:18,854 INFO MainThread:3148323 [wandb_init.py:_log_setup():508] Logging internal logs to /nfs/nfs2/home/sragas/demo/wandb/run-20230712_000318-e287bzfl/logs/debug-internal.log +2023-07-12 00:03:18,855 INFO MainThread:3148323 [wandb_init.py:init():547] calling init triggers +2023-07-12 00:03:18,855 INFO MainThread:3148323 [wandb_init.py:init():554] wandb.init called with sweep_config: {} +config: {} +2023-07-12 00:03:18,855 INFO MainThread:3148323 [wandb_init.py:init():591] wandb.init() called when a run is still active +2023-07-12 00:03:18,856 INFO MainThread:3148323 [wandb_run.py:_config_callback():1283] config_cb None None {'pretrained_model_name_or_path': 'runwayml/stable-diffusion-v1-5', 'revision': None, 'dataset_name': None, 'dataset_config_name': None, 'train_data_dir': '/l/vision/v5/sragas/easel_ai/thumbs_up_plain_dataset/', 'image_column': 'image', 'caption_column': 'text', 'validation_prompt': 'tom cruise thumbs up', 'num_validation_images': 4, 'validation_epochs': 1, 'max_train_samples': None, 'output_dir': '/l/vision/v5/sragas/easel_ai/models_plain/', 'cache_dir': '/l/vision/v5/sragas/hf_models/', 'seed': 51, 'resolution': 512, 'center_crop': True, 'random_flip': True, 'train_batch_size': 2, 'num_train_epochs': 300, 'max_train_steps': 3000, 'gradient_accumulation_steps': 4, 'gradient_checkpointing': False, 'learning_rate': 1e-05, 'scale_lr': False, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'dataloader_num_workers': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'push_to_hub': True, 'hub_token': None, 'prediction_type': None, 'hub_model_id': 'person-thumbs-up-plain-lora', 'logging_dir': 'logs', 'mixed_precision': None, 'report_to': 'wandb', 'local_rank': 0, 'checkpointing_steps': 500, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'enable_xformers_memory_efficient_attention': False, 'noise_offset': 0, 'rank': 4} +2023-07-12 12:35:49,765 WARNING MsgRouterThr:3148323 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb b/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..01aaa3e5c195f28a2a02fdd8a3035b8129403239 --- /dev/null +++ b/wandb/run-20230712_000308-yuype6tc/run-yuype6tc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5d829297a812d2211b0e8a7ab65f6cbf1438e028da37ecf91e36449c345009f +size 8366177